/* This file is part of the libmdbx amalgamated source code (v0.14.2-0-g530d0265 at 2026-05-14T21:14:59+03:00).
 *
 * libmdbx (aka MDBX) is an extremely fast, compact, powerful, embeddedable, transactional key-value storage engine with
 * open-source code. MDBX has a specific set of properties and capabilities, focused on creating unique lightweight
 * solutions.  Please visit https://libmdbx.dqdkfa.ru for more information, changelog, documentation, C++ API description
 * and links to the original git repo with the source code.  Questions, feedback and suggestions are welcome to the
 * Telegram' group https://t.me/libmdbx, MAX' chat https://max.ru/join/dKckvyuARxp1vRK-wnPur8zYCEkbR3OUOmpPWkWxp78.
 *
 * The libmdbx code will forever remain open and with high-quality free support, as far as the life circumstances of the
 * project participants allow. Donations are welcome to ETH `0xD104d8f8B2dC312aaD74899F83EBf3EEBDC1EA3A`,
 * BTC `bc1qzvl9uegf2ea6cwlytnanrscyv8snwsvrc0xfsu`, SOL `FTCTgbHajoLVZGr8aEFWMzx3NDMyS5wXJgfeMTmJznRi`.
 * Всё будет хорошо!
 *
 * For ease of use and to eliminate potential limitations in both distribution and obstacles in technology development,
 * libmdbx is distributed as an amalgamated source code starting at the end of 2025.  The source code of the tests, as
 * well as the internal documentation, will be available only to the team directly involved in the development.
 *
 * Copyright 2015-2026 Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru>
 * SPDX-License-Identifier: Apache-2.0
 *
 * For notes about the license change, credits and acknowledgments, please refer to the COPYRIGHT file. */

/* clang-format off */

/*----------------------------------------------------------------------------*/

#include "mdbx-internals.h"

#ifdef _MSC_VER
#pragma warning(push, 1)
#if _MSC_VER > 1913
#pragma warning(disable : 5054) /* deprecated between enumerations of different types */
#endif
#endif /* MSVC */

typedef struct dp dp_t;
typedef struct dpl dpl_t;
typedef struct kvx kvx_t;
typedef struct meta_ptr meta_ptr_t;
typedef struct inner_cursor subcur_t;
typedef struct cursor_couple cursor_couple_t;
typedef struct defer_free_item defer_free_item_t;

typedef struct troika {
  uint8_t fsm, recent, prefer_steady, tail_and_flags;
#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */
  uint32_t unused_pad;
#endif
#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7u)
#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64u)
#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128u)
#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3u)
  txnid_t txnid[NUM_METAS];
} troika_t;

typedef struct page_get_result {
  page_t *page;
  int err;
} pgr_t;

typedef struct bind_reader_slot_result {
  int err;
  reader_slot_t *slot;
} bsr_t;

#ifndef __cplusplus

MDBX_MAYBE_UNUSED static __always_inline void atomic_yield(void) {
#if defined(_WIN32) || defined(_WIN64)
  YieldProcessor();
#elif defined(__ia32__) || defined(__e2k__)
  __builtin_ia32_pause();
#elif defined(__ia64__)
#if defined(__HP_cc__) || defined(__HP_aCC__)
  _Asm_hint(_HINT_PAUSE);
#else
  __asm__ __volatile__("hint @pause");
#endif
#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || defined(__ARM_ARCH_6K__)
#ifdef __CC_ARM
  __yield();
#else
  __asm__ __volatile__("yield");
#endif
#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && __mips_isa_rev >= 2
  __asm__ __volatile__("pause");
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) ||        \
    defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi)
  __asm__ __volatile__(".word 0x00000140");
#else
  osal_yield();
#endif
}

#ifdef MDBX_HAVE_C11ATOMICS
#define osal_memory_fence(order, write) atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order))
#else /* MDBX_HAVE_C11ATOMICS */
#define osal_memory_fence(order, write)                                                                                \
  do {                                                                                                                 \
    osal_compiler_barrier();                                                                                           \
    if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease))                             \
      osal_memory_barrier();                                                                                           \
  } while (0)
#endif /* MDBX_HAVE_C11ATOMICS */

#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__)
#define atomic_store32(p, value, order)                                                                                \
  ({                                                                                                                   \
    const uint32_t value_to_store = (value);                                                                           \
    atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, mo_c11_store(order));                             \
    value_to_store;                                                                                                    \
  })
#define atomic_load32(p, order) atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order))
#define atomic_store64(p, value, order)                                                                                \
  ({                                                                                                                   \
    const uint64_t value_to_store = (value);                                                                           \
    atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, mo_c11_store(order));                             \
    value_to_store;                                                                                                    \
  })
#define atomic_load64(p, order) atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order))
#endif /* LCC && MDBX_HAVE_C11ATOMICS */

#ifndef atomic_store32
MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value,
                                                                 enum mdbx_memory_order order) {
  STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4);
#ifdef MDBX_HAVE_C11ATOMICS
  ASSERT(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
  atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order));
#else  /* MDBX_HAVE_C11ATOMICS */
  if (order != mo_Relaxed)
    osal_compiler_barrier();
  p->weak = value;
  osal_memory_fence(order, true);
#endif /* MDBX_HAVE_C11ATOMICS */
  return value;
}
#endif /* atomic_store32 */

#ifndef atomic_load32
MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32(const volatile mdbx_atomic_uint32_t *p,
                                                                enum mdbx_memory_order order) {
  STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4);
#ifdef MDBX_HAVE_C11ATOMICS
  ASSERT(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
  return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order));
#else  /* MDBX_HAVE_C11ATOMICS */
  osal_memory_fence(order, false);
  const uint32_t value = p->weak;
  if (order != mo_Relaxed)
    osal_compiler_barrier();
  return value;
#endif /* MDBX_HAVE_C11ATOMICS */
}
#endif /* atomic_load32 */

MDBX_MAYBE_UNUSED static __always_inline mdbx_pid_t atomic_load_pid(const volatile mdbx_atomic_uint32_t *p,
                                                                    enum mdbx_memory_order order) {
  STATIC_ASSERT(sizeof(mdbx_pid_t) == sizeof(uint32_t));
  return (mdbx_pid_t)atomic_load32(p, order);
}

MDBX_MAYBE_UNUSED static __always_inline mdbx_pid_t atomic_store_pid(mdbx_atomic_uint32_t *p, const mdbx_pid_t value,
                                                                     enum mdbx_memory_order order) {
  STATIC_ASSERT(sizeof(mdbx_pid_t) == sizeof(uint32_t));
  return (mdbx_pid_t)atomic_store32(p, value, order);
}

/*------------------------------------------------------------------------------
 * safe read/write volatile 64-bit fields on 32-bit architectures. */

/* LY: for testing non-atomic 64-bit txnid on 32-bit arches.
 * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */
#ifndef xMDBX_TXNID_STEP
#if MDBX_64BIT_CAS
#define xMDBX_TXNID_STEP 1u
#else
#define xMDBX_TXNID_STEP 2u
#endif
#endif /* xMDBX_TXNID_STEP */

#ifndef atomic_store64
MDBX_MAYBE_UNUSED static __always_inline uint64_t atomic_store64(mdbx_atomic_uint64_t *p, const uint64_t value,
                                                                 enum mdbx_memory_order order) {
  STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8);
#if MDBX_64BIT_ATOMIC
#if __GNUC_PREREQ(11, 0)
  STATIC_ASSERT(__alignof__(mdbx_atomic_uint64_t) >= sizeof(uint64_t));
#endif /* GNU C >= 11 */
#ifdef MDBX_HAVE_C11ATOMICS
  ASSERT(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
  atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order));
#else  /* MDBX_HAVE_C11ATOMICS */
  if (order != mo_Relaxed)
    osal_compiler_barrier();
  p->weak = value;
  osal_memory_fence(order, true);
#endif /* MDBX_HAVE_C11ATOMICS */
#else  /* !MDBX_64BIT_ATOMIC */
  osal_compiler_barrier();
  atomic_store32(&p->low, (uint32_t)value, mo_Relaxed);
  jitter4testing(true);
  atomic_store32(&p->high, (uint32_t)(value >> 32), order);
  jitter4testing(true);
#endif /* !MDBX_64BIT_ATOMIC */
  return value;
}
#endif /* atomic_store64 */

#ifndef atomic_load64
MDBX_MAYBE_UNUSED static
#if MDBX_64BIT_ATOMIC
    __always_inline
#endif /* MDBX_64BIT_ATOMIC */
        uint64_t atomic_load64(const volatile mdbx_atomic_uint64_t *p, enum mdbx_memory_order order) {
  STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8);
#if MDBX_64BIT_ATOMIC
#ifdef MDBX_HAVE_C11ATOMICS
  ASSERT(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p)));
  return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order));
#else  /* MDBX_HAVE_C11ATOMICS */
  osal_memory_fence(order, false);
  const uint64_t value = p->weak;
  if (order != mo_Relaxed)
    osal_compiler_barrier();
  return value;
#endif /* MDBX_HAVE_C11ATOMICS */
#else  /* !MDBX_64BIT_ATOMIC */
  osal_compiler_barrier();
  uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32;
  jitter4testing(true);
  value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease);
  jitter4testing(true);
  for (;;) {
    osal_compiler_barrier();
    uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32;
    jitter4testing(true);
    again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease);
    jitter4testing(true);
    if (likely(value == again))
      return value;
    value = again;
    atomic_yield();
  }
#endif /* !MDBX_64BIT_ATOMIC */
}
#endif /* atomic_load64 */

#if MDBX_64BIT_CAS
MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) {
#ifdef MDBX_HAVE_C11ATOMICS
  STATIC_ASSERT(sizeof(long long) >= sizeof(uint64_t));
  ASSERT(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
  return atomic_compare_exchange_strong(MDBX_c11a_rw(uint64_t, p), &c, v);
#elif defined(__GNUC__) || defined(__clang__)
  return __sync_bool_compare_and_swap(&p->weak, c, v);
#elif defined(_MSC_VER)
  return c == (uint64_t)_InterlockedCompareExchange64((volatile __int64 *)&p->weak, v, c);
#elif defined(__APPLE__)
  return OSAtomicCompareAndSwap64Barrier(c, v, &p->weak);
#else
#error FIXME: Unsupported compiler
#endif
}
#endif /* MDBX_64BIT_CAS */

MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) {
#ifdef MDBX_HAVE_C11ATOMICS
  STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
  ASSERT(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
  return atomic_compare_exchange_strong(MDBX_c11a_rw(uint32_t, p), &c, v);
#elif defined(__GNUC__) || defined(__clang__)
  return __sync_bool_compare_and_swap(&p->weak, c, v);
#elif defined(_MSC_VER)
  STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
  return c == (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c);
#elif defined(__APPLE__)
  return OSAtomicCompareAndSwap32Barrier(c, v, &p->weak);
#else
#error FIXME: Unsupported compiler
#endif
}

MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) {
#ifdef MDBX_HAVE_C11ATOMICS
  STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
  ASSERT(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
  return atomic_fetch_add(MDBX_c11a_rw(uint32_t, p), v);
#elif defined(__GNUC__) || defined(__clang__)
  return __sync_fetch_and_add(&p->weak, v);
#elif defined(_MSC_VER)
  STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
  return (uint32_t)_InterlockedExchangeAdd((volatile long *)&p->weak, v);
#elif defined(__APPLE__)
  return OSAtomicAdd32Barrier(v, &p->weak);
#else
#error FIXME: Unsupported compiler
#endif
}

#define atomic_sub32(p, v) atomic_add32(p, 0 - (v))

MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_txnid_next(uint64_t txnid) {
  txnid += xMDBX_TXNID_STEP;
#if !MDBX_64BIT_CAS
  /* avoid overflow of low-part in safe64_reset() */
  txnid += (UINT32_MAX == (uint32_t)txnid);
#endif
  return txnid;
}

/* Atomically make target value >= SAFE64_INVALID_THRESHOLD */
MDBX_MAYBE_UNUSED static __always_inline void safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) {
  if (single_writer) {
#if MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64
    atomic_store64(p, UINT64_MAX, mo_AcquireRelease);
#else
    atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease);
#endif /* MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64 */
  } else {
#if MDBX_64BIT_CAS && MDBX_64BIT_ATOMIC
    /* atomically make value >= SAFE64_INVALID_THRESHOLD by 64-bit operation */
    atomic_store64(p, UINT64_MAX, mo_AcquireRelease);
#elif MDBX_64BIT_CAS
    /* atomically make value >= SAFE64_INVALID_THRESHOLD by 32-bit operation */
    atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease);
#else
    /* it is safe to increment low-part to avoid ABA, since xMDBX_TXNID_STEP > 1
     * and overflow was preserved in safe64_txnid_next() */
    STATIC_ASSERT(xMDBX_TXNID_STEP > 1);
    atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */;
    atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease);
    atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */;
#endif /* MDBX_64BIT_CAS && MDBX_64BIT_ATOMIC */
  }
  ASSERT(p->weak >= SAFE64_INVALID_THRESHOLD);
  jitter4testing(true);
}

MDBX_MAYBE_UNUSED static __always_inline bool safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) {
  /* LY: This function is used to reset `txnid` from hsr-handler in case
   *     the asynchronously cancellation of read transaction. Therefore,
   *     there may be a collision between the cleanup performed here and
   *     asynchronous termination and restarting of the read transaction
   *     in another process/thread. In general we MUST NOT reset the `txnid`
   *     if a new transaction was started (i.e. if `txnid` was changed). */
#if MDBX_64BIT_CAS
  bool rc = atomic_cas64(p, compare, UINT64_MAX);
#else
  /* LY: There is no gold ratio here since shared mutex is too costly,
   *     in such way we must acquire/release it for every update of txnid,
   *     i.e. twice for each read transaction). */
  bool rc = false;
  if (likely(atomic_load32(&p->low, mo_AcquireRelease) == (uint32_t)compare &&
             atomic_cas32(&p->high, (uint32_t)(compare >> 32), UINT32_MAX))) {
    if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) != (uint32_t)compare))
      atomic_cas32(&p->high, UINT32_MAX, (uint32_t)(compare >> 32));
    else
      rc = true;
  }
#endif /* MDBX_64BIT_CAS */
  jitter4testing(true);
  return rc;
}

MDBX_MAYBE_UNUSED static __always_inline void safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) {
  ASSERT(p->weak >= SAFE64_INVALID_THRESHOLD);
#if MDBX_64BIT_ATOMIC && MDBX_64BIT_CAS
  atomic_store64(p, v, mo_AcquireRelease);
#else  /* MDBX_64BIT_ATOMIC */
  osal_compiler_barrier();
  /* update low-part but still value >= SAFE64_INVALID_THRESHOLD */
  atomic_store32(&p->low, (uint32_t)v, mo_Relaxed);
  ASSERT(p->weak >= SAFE64_INVALID_THRESHOLD);
  jitter4testing(true);
  /* update high-part from SAFE64_INVALID_THRESHOLD to actual value */
  atomic_store32(&p->high, (uint32_t)(v >> 32), mo_AcquireRelease);
#endif /* MDBX_64BIT_ATOMIC */
  ASSERT(p->weak == v);
  jitter4testing(true);
}

MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_read(const mdbx_atomic_uint64_t *p) {
  jitter4testing(true);
  uint64_t v = atomic_load64(p, mo_AcquireRelease);
  while (!MDBX_64BIT_ATOMIC && unlikely(v != p->weak)) {
    atomic_yield();
    v = atomic_load64(p, mo_AcquireRelease);
  }
  return v;
}

#if 0 /* unused for now */
MDBX_MAYBE_UNUSED static __always_inline bool safe64_is_valid(uint64_t v) {
#if MDBX_WORDBITS >= 64
  return v < SAFE64_INVALID_THRESHOLD;
#else
  return (v >> 32) != UINT32_MAX;
#endif /* MDBX_WORDBITS */
}

MDBX_MAYBE_UNUSED static __always_inline bool
 safe64_is_valid_ptr(const mdbx_atomic_uint64_t *p) {
#if MDBX_64BIT_ATOMIC
  return atomic_load64(p, mo_AcquireRelease) < SAFE64_INVALID_THRESHOLD;
#else
  return atomic_load32(&p->high, mo_AcquireRelease) != UINT32_MAX;
#endif /* MDBX_64BIT_ATOMIC */
}
#endif /* unused for now */

/* non-atomic write with safety for reading a half-updated value */
MDBX_MAYBE_UNUSED static __always_inline void safe64_update(mdbx_atomic_uint64_t *p, const uint64_t v) {
#if MDBX_64BIT_ATOMIC
  atomic_store64(p, v, mo_Relaxed);
#else
  safe64_reset(p, true);
  safe64_write(p, v);
#endif /* MDBX_64BIT_ATOMIC */
}

/* non-atomic increment with safety for reading a half-updated value */
MDBX_MAYBE_UNUSED static
#if MDBX_64BIT_ATOMIC
    __always_inline
#endif /* MDBX_64BIT_ATOMIC */
    void safe64_inc(mdbx_atomic_uint64_t *p, const uint64_t v) {
  ASSERT(v > 0);
  safe64_update(p, safe64_read(p) + v);
}

#endif /* !__cplusplus */

/* Сортированный набор txnid, использующий внутри комбинацию непрерывного интервала и списка.
 * Обеспечивает хранение id записей при переработке, очистку и обновлении GC, включая возврат остатков переработанных
 * страниц.
 *
 * При переработке GC записи преимущественно выбираются последовательно, но это не гарантируется. В LIFO-режиме
 * переработка и добавление записей в rkl происходит преимущественно в обратном порядке, но из-за завершения читающих
 * транзакций могут быть «скачки» в прямом направлении. В FIFO-режиме записи GC перерабатываются в прямом порядке и при
 * этом линейно, но не обязательно строго последовательно, при этом гарантируется что между добавляемыми в rkl
 * идентификаторами в GC нет записей, т.е. между первой (минимальный id) и последней (максимальный id) в GC нет записей
 * и весь интервал может быть использован для возврата остатков страниц в GC.
 *
 * Таким образом, комбинация линейного интервала и списка (отсортированного в порядке возрастания элементов) является
 * рациональным решением, близким к теоретически оптимальному пределу.
 *
 * Реализация rkl достаточно проста/прозрачная, если не считать неочевидную «магию» обмена непрерывного интервала и
 * образующихся в списке последовательностей. Однако, именно этот автоматически выполняемый без лишних операций обмен
 * оправдывает все накладные расходы. */
typedef struct MDBX_rkl {
  txnid_t solid_begin, solid_end; /* начало и конец непрерывной последовательности solid_begin ... solid_end-1. */
  unsigned list_length;           /* текущая длина списка. */
  unsigned list_limit;    /* размер буфера выделенного под список, равен ARRAY_LENGTH(inplace) когда list == inplace. */
  txnid_t *list;          /* список отдельных элементов в порядке возрастания (наименьший в начале). */
  txnid_t inplace[4 + 8]; /* статический массив для коротких списков, чтобы избавиться от выделения/освобождения памяти
                           * в большинстве случаев. */
} rkl_t;

#ifndef __cplusplus

MDBX_MAYBE_UNUSED MDBX_INTERNAL void rkl_init(rkl_t *rkl);
MDBX_MAYBE_UNUSED MDBX_INTERNAL int rkl_reserve(rkl_t *rkl, size_t wanna_size);
MDBX_MAYBE_UNUSED MDBX_INTERNAL void rkl_clear(rkl_t *rkl);
MDBX_MAYBE_UNUSED static inline void rkl_clear_and_shrink(rkl_t *rkl) { rkl_clear(rkl); /* TODO */ }
MDBX_MAYBE_UNUSED MDBX_INTERNAL void rkl_destroy(rkl_t *rkl);
MDBX_MAYBE_UNUSED MDBX_INTERNAL void rkl_destructive_move(rkl_t *src, rkl_t *dst);
MDBX_MAYBE_UNUSED MDBX_INTERNAL __must_check_result int rkl_copy(const rkl_t *src, rkl_t *dst);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool rkl_empty(const rkl_t *rkl) {
  return rkl->solid_begin > rkl->solid_end;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL bool rkl_check(const rkl_t *rkl);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t rkl_len(const rkl_t *rkl);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL txnid_t rkl_lowest(const rkl_t *rkl);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL txnid_t rkl_highest(const rkl_t *rkl);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t rkl_edge(const rkl_t *rkl,
                                                                            const bool highest_not_lowest) {
  return highest_not_lowest ? rkl_highest(rkl) : rkl_lowest(rkl);
}
MDBX_MAYBE_UNUSED MDBX_INTERNAL __must_check_result int rkl_push(rkl_t *rkl, const txnid_t id);
MDBX_MAYBE_UNUSED MDBX_INTERNAL txnid_t rkl_pop(rkl_t *rkl, const bool highest_not_lowest);
MDBX_MAYBE_UNUSED MDBX_INTERNAL __must_check_result int rkl_merge(const rkl_t *src, rkl_t *dst, bool ignore_duplicates);
MDBX_MAYBE_UNUSED MDBX_INTERNAL int rkl_destructive_merge(rkl_t *src, rkl_t *dst, bool ignore_duplicates);

/* Итератор для rkl.
 * Обеспечивает изоляцию внутреннего устройства rkl от остального кода, чем существенно его упрощает.
 * Фактически именно использованием rkl с итераторами ликвидируется "ребус" исторически образовавшийся в gc-update. */
typedef struct MDBX_rkl_iter {
  const rkl_t *rkl;
  unsigned pos;
  unsigned solid_offset;
} rkl_iter_t;

MDBX_MAYBE_UNUSED MDBX_INTERNAL __must_check_result rkl_iter_t rkl_iterator(const rkl_t *rkl, const bool reverse);
MDBX_MAYBE_UNUSED MDBX_INTERNAL __must_check_result txnid_t rkl_turn(rkl_iter_t *iter, const bool reverse);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t rkl_left(rkl_iter_t *iter, const bool reverse);
MDBX_MAYBE_UNUSED MDBX_INTERNAL bool rkl_find(const rkl_t *rkl, const txnid_t id, rkl_iter_t *iter);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __must_check_result MDBX_INTERNAL bool rkl_contain(const rkl_t *rkl,
                                                                                                txnid_t id);

typedef struct MDBX_rkl_hole {
  txnid_t begin;
  txnid_t end;
} rkl_hole_t;
MDBX_MAYBE_UNUSED MDBX_INTERNAL __must_check_result rkl_hole_t rkl_hole(rkl_iter_t *iter, const bool reverse);

#endif /* !__cplusplus */

/* List of txnid */
typedef txnid_t *txl_t;
typedef const txnid_t *const_txl_t;

enum txl_rules {
  txl_granulate = 32,
  txl_initial = txl_granulate - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t),
  txl_max = (1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)
};

#ifndef __cplusplus

MDBX_MAYBE_UNUSED MDBX_INTERNAL txl_t txl_alloc(void);

MDBX_MAYBE_UNUSED MDBX_INTERNAL void txl_free(txl_t txl);

MDBX_MAYBE_UNUSED MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id);

MDBX_MAYBE_UNUSED MDBX_INTERNAL void txl_sort(txl_t txl);

MDBX_MAYBE_UNUSED MDBX_INTERNAL bool txl_contain(const txl_t txl, txnid_t id);

MDBX_MAYBE_UNUSED static inline size_t txl_alloclen(const_txl_t txl) { return txl[-1]; }

MDBX_MAYBE_UNUSED static inline size_t txl_size(const_txl_t txl) { return txl[0]; }

#endif /* !__cplusplus */

/*------------------------------------------------------------------------------
 * Unaligned access */

MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t field_alignment(size_t alignment_baseline,
                                                                                   size_t field_offset) {
  size_t merge = alignment_baseline | (size_t)field_offset;
  return merge & -(int)merge;
}

/* read-thunk for UB-sanitizer */
MDBX_NOTHROW_PURE_FUNCTION MDBX_MAYBE_UNUSED static inline uint8_t peek_u8(const uint8_t *__restrict ptr) {
  return *ptr;
}

/* write-thunk for UB-sanitizer */
MDBX_MAYBE_UNUSED static inline void poke_u8(uint8_t *__restrict ptr, const uint8_t v) { *ptr = v; }

MDBX_MAYBE_UNUSED static inline void *bcopy_2(void *__restrict dst, const void *__restrict src) {
  uint8_t *__restrict d = (uint8_t *)dst;
  const uint8_t *__restrict s = (uint8_t *)src;
  d[0] = s[0];
  d[1] = s[1];
  return d;
}

MDBX_MAYBE_UNUSED static inline void *bcopy_4(void *const __restrict dst, const void *const __restrict src) {
  uint8_t *__restrict d = (uint8_t *)dst;
  const uint8_t *__restrict s = (uint8_t *)src;
  d[0] = s[0];
  d[1] = s[1];
  d[2] = s[2];
  d[3] = s[3];
  return d;
}

MDBX_MAYBE_UNUSED static inline void *bcopy_8(void *const __restrict dst, const void *const __restrict src) {
  uint8_t *__restrict d = (uint8_t *)dst;
  const uint8_t *__restrict s = (uint8_t *)src;
  d[0] = s[0];
  d[1] = s[1];
  d[2] = s[2];
  d[3] = s[3];
  d[4] = s[4];
  d[5] = s[5];
  d[6] = s[6];
  d[7] = s[7];
  return d;
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_MAYBE_UNUSED static inline uint16_t unaligned_peek_u16(const size_t expected_alignment,
                                                                                       const void *const ptr) {
  ASSERT((uintptr_t)ptr % expected_alignment == 0);
  if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(uint16_t)) == 0)
    return *(const uint16_t *)ptr;
  else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
    return *(const __unaligned uint16_t *)ptr;
#else
    uint16_t v;
    bcopy_2((uint8_t *)&v, (const uint8_t *)ptr);
    return v;
#endif /* _MSC_VER || __unaligned */
  }
}

MDBX_MAYBE_UNUSED static inline void unaligned_poke_u16(const size_t expected_alignment, void *const __restrict ptr,
                                                        const uint16_t v) {
  ASSERT((uintptr_t)ptr % expected_alignment == 0);
  if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(v)) == 0)
    *(uint16_t *)ptr = v;
  else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
    *((uint16_t __unaligned *)ptr) = v;
#else
    bcopy_2((uint8_t *)ptr, (const uint8_t *)&v);
#endif /* _MSC_VER || __unaligned */
  }
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_MAYBE_UNUSED static inline uint32_t
unaligned_peek_u32(const size_t expected_alignment, const void *const __restrict ptr) {
  ASSERT((uintptr_t)ptr % expected_alignment == 0);
  if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(uint32_t)) == 0)
    return *(const uint32_t *)ptr;
  else if ((expected_alignment % sizeof(uint16_t)) == 0) {
    const uint16_t lo = ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
    const uint16_t hi = ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
    return lo | (uint32_t)hi << 16;
  } else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
    return *(const __unaligned uint32_t *)ptr;
#else
    uint32_t v;
    bcopy_4((uint8_t *)&v, (const uint8_t *)ptr);
    return v;
#endif /* _MSC_VER || __unaligned */
  }
}

MDBX_MAYBE_UNUSED static inline void unaligned_poke_u32(const size_t expected_alignment, void *const __restrict ptr,
                                                        const uint32_t v) {
  ASSERT((uintptr_t)ptr % expected_alignment == 0);
  if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(v)) == 0)
    *(uint32_t *)ptr = v;
  else if ((expected_alignment % sizeof(uint16_t)) == 0) {
    ((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v;
    ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint16_t)(v >> 16);
  } else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
    *((uint32_t __unaligned *)ptr) = v;
#else
    bcopy_4((uint8_t *)ptr, (const uint8_t *)&v);
#endif /* _MSC_VER || __unaligned */
  }
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_MAYBE_UNUSED static inline uint64_t
unaligned_peek_u64(const size_t expected_alignment, const void *const __restrict ptr) {
  ASSERT((uintptr_t)ptr % expected_alignment == 0);
  if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
    return *(const uint64_t *)ptr;
  else if ((expected_alignment % sizeof(uint32_t)) == 0) {
    const uint32_t lo = ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
    const uint32_t hi = ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
    return lo | (uint64_t)hi << 32;
  } else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
    return *(const __unaligned uint64_t *)ptr;
#else
    uint64_t v;
    bcopy_8((uint8_t *)&v, (const uint8_t *)ptr);
    return v;
#endif /* _MSC_VER || __unaligned */
  }
}

MDBX_MAYBE_UNUSED static inline uint64_t unaligned_peek_u64_volatile(const size_t expected_alignment,
                                                                     const volatile void *const __restrict ptr) {
  ASSERT((uintptr_t)ptr % expected_alignment == 0);
  ASSERT(expected_alignment % sizeof(uint32_t) == 0);
  if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
    return *(const volatile uint64_t *)ptr;
  else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
    return *(const volatile __unaligned uint64_t *)ptr;
#else
    const uint32_t lo = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
    const uint32_t hi = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
    return lo | (uint64_t)hi << 32;
#endif /* _MSC_VER || __unaligned */
  }
}

MDBX_MAYBE_UNUSED static inline void unaligned_poke_u64(const size_t expected_alignment, void *const __restrict ptr,
                                                        const uint64_t v) {
  ASSERT((uintptr_t)ptr % expected_alignment == 0);
  if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(v)) == 0)
    *(uint64_t *)ptr = v;
  else if ((expected_alignment % sizeof(uint32_t)) == 0) {
    ((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v;
    ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint32_t)(v >> 32);
  } else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
    *((uint64_t __unaligned *)ptr) = v;
#else
    bcopy_8((uint8_t *)ptr, (const uint8_t *)&v);
#endif /* _MSC_VER || __unaligned */
  }
}

#define UNALIGNED_PEEK_8(ptr, struct, field) peek_u8(ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_8(ptr, struct, field, value) poke_u8(ptr_disp(ptr, offsetof(struct, field)), value)

#define UNALIGNED_PEEK_16(ptr, struct, field) unaligned_peek_u16(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_16(ptr, struct, field, value)                                                                   \
  unaligned_poke_u16(1, ptr_disp(ptr, offsetof(struct, field)), value)

#define UNALIGNED_PEEK_32(ptr, struct, field) unaligned_peek_u32(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_32(ptr, struct, field, value)                                                                   \
  unaligned_poke_u32(1, ptr_disp(ptr, offsetof(struct, field)), value)

#define UNALIGNED_PEEK_64(ptr, struct, field) unaligned_peek_u64(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_64(ptr, struct, field, value)                                                                   \
  unaligned_poke_u64(1, ptr_disp(ptr, offsetof(struct, field)), value)

MDBX_NOTHROW_PURE_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t peek_pgno(const void *const __restrict ptr) {
  if (sizeof(pgno_t) == sizeof(uint32_t))
    return (pgno_t)unaligned_peek_u32(1, ptr);
  else if (sizeof(pgno_t) == sizeof(uint64_t))
    return (pgno_t)unaligned_peek_u64(1, ptr);
  else {
    pgno_t pgno;
    memcpy(&pgno, ptr, sizeof(pgno));
    return pgno;
  }
}

MDBX_MAYBE_UNUSED static inline void poke_pgno(void *const __restrict ptr, const pgno_t pgno) {
  if (sizeof(pgno) == sizeof(uint32_t))
    unaligned_poke_u32(1, ptr, pgno);
  else if (sizeof(pgno) == sizeof(uint64_t))
    unaligned_poke_u64(1, ptr, pgno);
  else
    memcpy(ptr, &pgno, sizeof(pgno));
}

#if defined(_WIN32) || defined(_WIN64)

typedef union osal_srwlock {
  __anonymous_struct_extension__ struct {
    long volatile readerCount;
    long volatile writerCount;
  };
  RTL_SRWLOCK native;
} osal_srwlock_t;

typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *);

#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */
typedef enum _FILE_INFO_BY_HANDLE_CLASS {
  FileBasicInfo,
  FileStandardInfo,
  FileNameInfo,
  FileRenameInfo,
  FileDispositionInfo,
  FileAllocationInfo,
  FileEndOfFileInfo,
  FileStreamInfo,
  FileCompressionInfo,
  FileAttributeTagInfo,
  FileIdBothDirectoryInfo,
  FileIdBothDirectoryRestartInfo,
  FileIoPriorityHintInfo,
  FileRemoteProtocolInfo,
  MaximumFileInfoByHandleClass
} FILE_INFO_BY_HANDLE_CLASS,
    *PFILE_INFO_BY_HANDLE_CLASS;

typedef struct _FILE_END_OF_FILE_INFO {
  LARGE_INTEGER EndOfFile;
} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO;

#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001
#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002

typedef struct _FILE_REMOTE_PROTOCOL_INFO {
  USHORT StructureVersion;
  USHORT StructureSize;
  DWORD Protocol;
  USHORT ProtocolMajorVersion;
  USHORT ProtocolMinorVersion;
  USHORT ProtocolRevision;
  USHORT Reserved;
  DWORD Flags;
  struct {
    DWORD Reserved[8];
  } GenericReserved;
  struct {
    DWORD Reserved[16];
  } ProtocolSpecificReserved;
} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO;

#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */

typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(_In_ HANDLE hFile,
                                                        _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
                                                        _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);

typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
    _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, _In_ DWORD nVolumeNameSize,
    _Out_opt_ LPDWORD lpVolumeSerialNumber, _Out_opt_ LPDWORD lpMaximumComponentLength,
    _Out_opt_ LPDWORD lpFileSystemFlags, _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);

typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, _Out_ LPWSTR lpszFilePath,
                                                      _In_ DWORD cchFilePath, _In_ DWORD dwFlags);

typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(_In_ HANDLE hFile,
                                                      _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
                                                      _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);

typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(IN HANDLE FileHandle, IN OUT HANDLE Event,
                                              IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
                                              OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
                                              IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
                                              OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);

typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);

#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
typedef struct _WIN32_MEMORY_RANGE_ENTRY {
  PVOID VirtualAddress;
  SIZE_T NumberOfBytes;
} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
#endif /* Windows 8.x */

typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(HANDLE hProcess, ULONG_PTR NumberOfEntries,
                                                 PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);

typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;

typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, IN PLARGE_INTEGER NewSectionSize);

typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, LPCSTR lpValue, DWORD dwFlags, LPDWORD pdwType,
                                           PVOID pvData, LPDWORD pcbData);

typedef long(WINAPI *MDBX_CoCreateGuid)(bin128_t *guid);

NTSYSAPI ULONG RtlRandomEx(PULONG Seed);

typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, PUCHAR OverlappedRangeStart, ULONG Length);

struct libmdbx_imports {
  osal_srwlock_t_function srwl_Init;
  osal_srwlock_t_function srwl_AcquireShared;
  osal_srwlock_t_function srwl_ReleaseShared;
  osal_srwlock_t_function srwl_AcquireExclusive;
  osal_srwlock_t_function srwl_ReleaseExclusive;
  MDBX_NtExtendSection NtExtendSection;
  MDBX_GetFileInformationByHandleEx GetFileInformationByHandleEx;
  MDBX_GetVolumeInformationByHandleW GetVolumeInformationByHandleW;
  MDBX_GetFinalPathNameByHandleW GetFinalPathNameByHandleW;
  MDBX_SetFileInformationByHandle SetFileInformationByHandle;
  MDBX_NtFsControlFile NtFsControlFile;
  MDBX_PrefetchVirtualMemory PrefetchVirtualMemory;
  MDBX_GetTickCount64 GetTickCount64;
  MDBX_RegGetValueA RegGetValueA;
  MDBX_SetFileIoOverlappedRange SetFileIoOverlappedRange;
  MDBX_CoCreateGuid CoCreateGuid;
};

#ifndef __cplusplus
MDBX_INTERNAL void windows_import(void);
#endif /* !__cplusplus */
#endif /* Windows */

enum signatures {
  env_signature = INT32_C(0x1A899641),
  txn_signature = INT32_C(0x13D53A31),
  cur_signature_live = INT32_C(0x7E05D5B1),
  cur_signature_ready4dispose = INT32_C(0x2817A047),
  cur_signature_wait4eot = INT32_C(0x10E297A7)
};

/*----------------------------------------------------------------------------*/

/* An dirty-page list item is an pgno/pointer pair. */
struct dp {
  page_t *ptr;
  pgno_t pgno;
#if MDBX_DPL_CACHE_NPAGES
  pgno_t npages;
#endif /* MDBX_DPL_CACHE_NPAGES */
};

enum dpl_rules {
  dpl_gap_edging = 2,
  dpl_gap_mergesort = 16,
  dpl_reserve_gap = dpl_gap_mergesort + dpl_gap_edging,
  dpl_insertion_threshold = 42
};

/* An DPL (dirty-page list) is a lazy-sorted array of MDBX_DPs. */
struct dpl {
  size_t sorted;
  size_t length;
  /* number of pages, but not an entries. */
  size_t pages_including_loose;
  /* allocated size excluding the dpl_reserve_gap */
  size_t detent;
  /* dynamic size with holes at zero and after the last */
  dp_t items[dpl_reserve_gap];
};

/*----------------------------------------------------------------------------*/
/* Internal structures */

typedef struct search_foliage_result {
  node_t *node;
  bool exact;
} sfr_t;

typedef size_t (*MDBX_search_branch)(const MDBX_cursor *mc, const MDBX_val *key);
typedef sfr_t (*MDBX_search_foliage)(MDBX_cursor *mc, const MDBX_val *key);

/* Comparing/ordering and length constraints */
typedef struct clc {
  MDBX_cmp_func cmp; /* comparator */
  size_t lmin, lmax; /* min/max length constraints */
  MDBX_search_branch search_branch;
  MDBX_search_foliage search_foliage;
  void *reserve_node_add;
  void *reserve_node_del;
} clc_t;

/* Вспомогательная информация о table.
 *
 * Совокупность потребностей:
 * 1. Для транзакций и основного курсора нужны все поля.
 * 2. Для вложенного dupsort-курсора нужен компаратор значений, который изнутри
 *    курсора будет выглядеть как компаратор ключей. Плюс заглушка компаратора
 *    значений, которая не должна использоваться в штатных ситуациях, но
 *    требуется хотя-бы для отслеживания таких обращений.
 * 3. Использование компараторов для курсора и вложенного dupsort-курсора
 *    должно выглядеть одинаково.
 * 4. Желательно минимизировать объём данных размещаемых внутри вложенного
 *    dupsort-курсора.
 * 5. Желательно чтобы объем всей структуры был степенью двойки.
 *
 * Решение:
 *  - не храним в dupsort-курсоре ничего лишнего, а только tree;
 *  - в курсоры помещаем только указатель на clc_t, который будет указывать
 *    на соответствующее clc-поле в общей kvx-таблице привязанной к env;
 *  - компаратор размещаем в начале clc_t, в kvx_t сначала размещаем clc
 *    для ключей, потом для значений, а имя БД в конце kvx_t.
 *  - тогда в курсоре clc[0] будет содержать информацию для ключей,
 *    а clc[1] для значений, причем компаратор значений для dupsort-курсора
 *    будет попадать на MDBX_val с именем, что приведет к SIGSEGV при попытке
 *    использования такого компаратора.
 *  - размер kvx_t становится равным 8 словам (16 словам после добавление функций поиска и т.п).
 *
 * Трюки и прочая экономия на спичках:
 *  - не храним dbi внутри курсора, вместо этого вычисляем его как разницу между
 *    dbi_state курсора и началом таблицы dbi_state в транзакции. Смысл тут в
 *    экономии кол-ва полей при инициализации курсора. Затрат это не создает,
 *    так как dbi требуется для последующего доступа к массивам в транзакции,
 *    т.е. при вычислении dbi разыменовывается тот-же указатель на txn
 *    и читается та же кэш-линия с указателями. */
typedef struct clc_couple {
  clc_t k; /* для ключей */
  clc_t v; /* для значений */
} clc_couple_t;

struct kvx {
  clc_couple_t clc;
  MDBX_val name; /* имя table */
};

/* Non-shared DBI state flags inside transaction */
enum dbi_state {
  DBI_DIRTY = 0x01 /* table was written in this txn */,
  DBI_STALE = 0x02 /* cached table record is outdated and should be reloaded/refreshed */,
  DBI_FRESH = 0x04 /* table handle opened in this txn */,
  DBI_CREAT = 0x08 /* table handle created in this txn */,
  DBI_VALID = 0x10 /* Handle is valid, see also DB_VALID */,
  DBI_OLDEN = 0x40 /* Handle was closed/reopened outside txn */,
  DBI_LINDO = 0x80 /* Lazy initialization done for DBI-slot */,
};

enum txn_flags {
  txn_ro_flat = MDBX_TXN_RDONLY,
  txn_ro_nested = UINT32_C(0x0800),
  txn_ro_both = txn_ro_flat | txn_ro_nested,
  txn_ro_begin_flags = MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE,
  txn_rw_begin_flags = MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY,
  txn_rw_already_locked = MDBX_TXN_RDONLY_PREPARE & ~MDBX_TXN_RDONLY,
  txn_shrink_allowed = UINT32_C(0x40000000),
  txn_parked = MDBX_TXN_PARKED,
  txn_gc_drained = 0x100 /* GC was depleted up to oldest reader */,
  txn_may_have_cursors = 0x400,
  txn_state_flags = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD |
                    MDBX_TXN_INVALID | txn_gc_drained
};

/* A database transaction.
 * Every operation requires a transaction handle. */
struct MDBX_txn {
  int32_t signature;
  uint32_t flags; /* Transaction Flags */
  size_t n_dbi;
  size_t owner; /* thread ID that owns this transaction */

  MDBX_txn *parent; /* parent of a nested txn */
  MDBX_txn *nested; /* nested txn under this txn,
                       set together with MDBX_TXN_HAS_CHILD */
  geo_t geo;

  /* The ID of this transaction. IDs are integers incrementing from
   * INITIAL_TXNID. Only committed write transactions increment the ID. If a
   * transaction aborts, the ID may be re-used by the next writer. */
  txnid_t txnid, front_txnid;

  MDBX_env *env; /* the DB environment */
  tree_t *dbs;   /* Array of tree_t records for each known DB */

#if MDBX_ENABLE_DBI_SPARSE
  unsigned *__restrict dbi_sparse;
#endif /* MDBX_ENABLE_DBI_SPARSE */

  /* Array of non-shared txn's flags of DBI.
   * Модификатор __restrict тут полезен и безопасен в текущем понимании,
   * так как пересечение возможно только с dbi_state курсоров,
   * и происходит по-чтению до последующего изменения/записи. */
  uint8_t *__restrict dbi_state;

  /* Array of sequence numbers for each DB handle. */
  uint32_t *__restrict dbi_seqs;

  /* Массив с головами односвязных списков отслеживания курсоров. */
  MDBX_cursor **cursors;

  /* "Канареечные" маркеры/счетчики */
  MDBX_canary canary;

  /* User-settable context */
  void *userctx;

#if MDBX_ENABLE_PGET_STAT
  /* Counter of page get operations */
  uint64_t ops_pget;
#endif /* MDBX_ENABLE_PGET_STAT */

  union {
    struct {
      /* For read txns: This thread/txn's slot table slot, or nullptr. */
      reader_slot_t *slot;
    } ro;
    struct {
      troika_t troika;
      pnl_t __restrict repnl; /* Reclaimed GC pages */
      struct {
        rkl_t reclaimed;   /* The list of reclaimed txn-ids from GC, but not cleared/deleted */
        rkl_t ready4reuse; /* The list of reclaimed txn-ids from GC, and cleared/deleted */
        uint64_t spent;    /* Time spent reading and searching GC */
        rkl_t comeback;    /* The list of ids of records returned into GC during commit, etc */
      } gc;
      bool prefault_write_activated;
#if MDBX_ENABLE_REFUND
      pgno_t loose_refund_wl /* FIXME: describe */;
#endif /* MDBX_ENABLE_REFUND */
      /* a sequence to spilling dirty page with LRU policy */
      unsigned dirtylru;
      /* dirtylist room: Dirty array size - dirty pages visible to this txn.
       * Includes ancestor txns' dirty pages not hidden by other txns'
       * dirty/spilled pages. Thus commit(nested txn) has room to merge
       * dirtylist into parent after freeing hidden parent pages. */
      size_t dirtyroom;
      /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */
      dpl_t *__restrict dirtylist;
      /* The list of pages that became unused during this transaction. */
      pnl_t __restrict retired_pages;
      /* The list of loose pages that became unused and may be reused
       * in this transaction, linked through `page_next()`. */
      page_t *__restrict loose_pages;
      /* Number of loose pages (wr.loose_pages) */
      size_t loose_count;
      union {
        struct {
          size_t least_removed;
          /* The sorted list of dirty pages we temporarily wrote to disk
           * because the dirty list was full. page numbers in here are
           * shifted left by 1, deleted slots have the LSB set. */
          pnl_t __restrict list;
        } spilled;
        size_t writemap_dirty_npages;
        size_t writemap_spilled_npages;
      };
      void *preserve_parent_userctx;
      /* In write txns, next is located the array of cursors for each DB */
    } wr;
  };
};

#define CURSOR_STACK_SIZE (16 + MDBX_WORDBITS / 4)

struct MDBX_cursor {
  int32_t signature;
  union {
    /* Тут некоторые трюки/заморочки с тем чтобы во всех основных сценариях
     * проверять состояние курсора одной простой операцией сравнения,
     * и при этом ни на каплю не усложнять код итерации стека курсора.
     *
     * Поэтому решение такое:
     *  - поля flags и top сделаны знаковыми, а их отрицательные значения
     *    используются для обозначения не-установленного/не-инициализированного
     *    состояния курсора;
     *  - для инвалидации/сброса курсора достаточно записать отрицательное
     *    значение в объединенное поле top_and_flags;
     *  - все проверки состояния сводятся к сравнению одного из полей
     *    flags/snum/snum_and_flags, которые в зависимости от сценария,
     *    трактуются либо как знаковые, либо как безнаковые. */
    __anonymous_struct_extension__ struct {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
      int8_t flags;
      /* индекс вершины стека, меньше нуля для не-инициализированного курсора */
      int8_t top;
#else
      int8_t top;
      int8_t flags;
#endif
    };
    int16_t top_and_flags;
  };
  /* флаги проверки, в том числе биты для проверки типа листовых страниц. */
  uint8_t checking;
  uint8_t pad;

  /* Указывает на txn->dbi_state[] для DBI этого курсора.
   * Модификатор __restrict тут полезен и безопасен в текущем понимании,
   * так как пересечение возможно только с dbi_state транзакции,
   * и происходит по-чтению до последующего изменения/записи. */
  uint8_t *__restrict dbi_state;
  /* Связь списка отслеживания курсоров в транзакции. */
  MDBX_txn *txn;
  /* Указывает на tree->dbs[] для DBI этого курсора. */
  tree_t *tree;
  /* Указывает на env->kvs[] для DBI этого курсора. */
  clc_couple_t *clc;
  subcur_t *__restrict subcur;
  page_t *pg[CURSOR_STACK_SIZE]; /* stack of pushed pages */
  indx_t ki[CURSOR_STACK_SIZE];  /* stack of page indices */
  MDBX_cursor *next;
  /* Состояние на момент старта вложенной транзакции */
  MDBX_cursor *backup;
#ifndef MDBX_DEBUG_SEARCH_DISPATCHING
#define MDBX_DEBUG_SEARCH_DISPATCHING MDBX_DEBUG
#endif /* MDBX_DEBUG_SEARCH_DISPATCHING */

#if MDBX_DEBUG_SEARCH_DISPATCHING
  unsigned search_step_counter;
#define MDBX_CURSOR_STC_INC(cursor)                                                                                    \
  do                                                                                                                   \
    ((MDBX_cursor *)(cursor))->search_step_counter += 1;                                                               \
  while (0)
#define MDBX_CURSOR_STC_GET(cursor) ((cursor)->search_step_counter)
#else
#define MDBX_CURSOR_STC_INC(cursor) __noop
#define MDBX_CURSOR_STC_GET(cursor) (0)
#endif /* MDBX_DEBUG_SEARCH_DISPATCHING */
};

struct inner_cursor {
  MDBX_cursor cursor;
  tree_t nested_tree;
};

struct cursor_couple {
  MDBX_cursor outer;
  void *userctx; /* User-settable context */
  subcur_t inner;
};

enum env_flags {
  /* Failed to update the meta page. Probably an I/O error. */
  ENV_FATAL_ERROR = INT32_MIN /* 0x80000000 */,
  /* Some fields are initialized. */
  ENV_ACTIVE = UINT32_C(0x20000000),
  /* me_txkey is set */
  ENV_TXKEY = UINT32_C(0x10000000),
  /* Legacy MDBX_MAPASYNC (prior v0.9) */
  DEPRECATED_MAPASYNC = UINT32_C(0x100000),
  /* Legacy MDBX_COALESCE (prior v0.12) */
  DEPRECATED_COALESCE = UINT32_C(0x2000000),
  ENV_INTERNAL_FLAGS = ENV_FATAL_ERROR | ENV_ACTIVE | ENV_TXKEY,
  /* Only a subset of the mdbx_env flags can be changed
   * at runtime. Changing other flags requires closing the
   * environment and re-opening it with the new flags. */
  ENV_CHANGEABLE_FLAGS = MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_MAPASYNC | MDBX_NOMEMINIT |
                         DEPRECATED_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | MDBX_VALIDATION,
  ENV_CHANGELESS_FLAGS = MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS | MDBX_NORDAHEAD |
                         MDBX_LIFORECLAIM | MDBX_EXCLUSIVE,
  ENV_USABLE_FLAGS = ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS
};

/* The database environment. */
struct MDBX_env {
  /* ----------------------------------------------------- mostly static part */
  mdbx_atomic_uint32_t signature;
  uint32_t flags;
  unsigned ps;          /* DB page size, initialized from me_os_psize */
  osal_mmap_t dxb_mmap; /* The main data file */
#define lazy_fd dxb_mmap.fd
  mdbx_filehandle_t dsync_fd, fd4meta;
#if defined(_WIN32) || defined(_WIN64)
  HANDLE dxb_lock_event;
  HANDLE lck_lock_event;
#endif                  /* Windows */
  osal_mmap_t lck_mmap; /* The lock file */
  lck_t *lck;

  uint16_t leaf_nodemax;   /* max size of a leaf-node */
  uint16_t branch_nodemax; /* max size of a branch-node */
  uint16_t subpage_limit;
  uint16_t subpage_room_threshold;
  uint16_t subpage_reserve_prereq;
  uint16_t subpage_reserve_limit;
  atomic_pgno_t mlocked_pgno;
  uint8_t ps2ln;              /* log2 of DB page size */
  int8_t stuck_meta;          /* recovery-only: target meta page or less that zero */
  uint16_t merge_threshold;   /* pages emptier than this are candidates for merging */
  unsigned max_readers;       /* size of the reader table */
  MDBX_dbi max_dbi;           /* size of the DB table */
  mdbx_pid_t pid;             /* process ID of this env */
  osal_thread_key_t me_txkey; /* thread-key for readers */
  struct {                    /* path to the DB files */
    pathchar_t *lck, *dxb, *specified;
    void *buffer;
  } pathname;
  void *page_auxbuf;              /* scratch area for DUPSORT put() */
  MDBX_txn *basal_txn;            /* preallocated write transaction */
  kvx_t *kvs;                     /* array of auxiliary key-value properties */
  uint8_t *__restrict dbs_flags;  /* array of flags from tree_t.flags */
  mdbx_atomic_uint32_t *dbi_seqs; /* array of dbi sequence numbers */
  unsigned maxgc_large1page;      /* Number of pgno_t fit in a single large page */
  unsigned maxgc_per_branch;
  mdbx_pid_t registered_reader_pid; /* have liveness lock in reader table */
  void *userctx;                    /* User-settable context */
  MDBX_hsr_func hsr_callback;       /* Callback for kicking laggard readers */
  size_t madv_threshold;

  struct {
    unsigned dp_reserve_limit;
    unsigned rp_augment_limit;
    unsigned dp_limit;
    unsigned dp_initial;
    uint64_t gc_time_limit;
    uint8_t dp_loose_limit;
    uint8_t spill_max_denominator;
    uint8_t spill_min_denominator;
    uint8_t spill_parent4child_denominator;
    uint16_t merge_threshold_dot16;
    uint16_t split_reserve_dot16;
#if !(defined(_WIN32) || defined(_WIN64))
    unsigned writethrough_threshold;
#endif /* Windows */
    bool prefault_write;
    bool prefer_waf_insteadof_balance; /* Strive to minimize WAF instead of
                                          balancing pages fullment */
    bool need_dp_limit_adjust;
    struct {
      uint16_t limit;
      uint16_t room_threshold;
      uint16_t reserve_prereq;
      uint16_t reserve_limit;
    } subpage;

    union {
      unsigned all;
      /* tracks options with non-auto values but tuned by user */
      struct {
        unsigned dp_limit : 1;
        unsigned rp_augment_limit : 1;
        unsigned prefault_write : 1;
      } non_auto;
    } flags;
  } options;

  /* struct geo_in_bytes used for accepting db-geo params from user for the new
   * database creation, i.e. when mdbx_env_set_geometry() was called before
   * mdbx_env_open(). */
  struct {
    size_t lower;  /* minimal size of datafile */
    size_t upper;  /* maximal size of datafile */
    size_t now;    /* current size of datafile */
    size_t grow;   /* step to grow datafile */
    size_t shrink; /* threshold to shrink datafile */
  } geo_in_bytes;

#if MDBX_LOCKING == MDBX_LOCKING_SYSV
  union {
    key_t key;
    int semid;
  } me_sysv_ipc;
#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */
  bool incore;

#if MDBX_ENABLE_DBI_LOCKFREE
  defer_free_item_t *defer_free;
#endif /* MDBX_ENABLE_DBI_LOCKFREE */

  /* -------------------------------------------------------------- debugging */

#ifdef ENABLE_MEMCHECK
  int valgrind_handle;
#endif
#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
  pgno_t poison_edge;
#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */

#ifndef xMDBX_DEBUG_SPILLING
#define xMDBX_DEBUG_SPILLING 0
#endif
#if xMDBX_DEBUG_SPILLING == 2
  size_t debug_dirtied_est, debug_dirtied_act;
#endif /* xMDBX_DEBUG_SPILLING */

  /* --------------------------------------------------- mostly volatile part */

  MDBX_txn *txn; /* current write transaction */
  struct {
    txnid_t detent;
  } gc;
  osal_fastmutex_t dbi_lock;
  unsigned n_dbi; /* number of DBs opened */

  unsigned shadow_reserve_len;
  page_t *__restrict shadow_reserve; /* list of malloc'ed blocks for re-use */

  osal_ioring_t ioring;

#if defined(_WIN32) || defined(_WIN64)
  osal_srwlock_t remap_lock;
  /* Workaround for LockFileEx and WriteFile multithread bug */
  CRITICAL_SECTION lck_event_cs;
  CRITICAL_SECTION dxb_event_cs;
  char *pathname_char; /* cache of multi-byte representation of pathname
                             to the DB files */
#else
  osal_fastmutex_t remap_lock;
#endif

  /* ------------------------------------------------- stub for lck-less mode */
  mdbx_atomic_uint64_t lckless_placeholder[(sizeof(lck_t) + MDBX_CACHELINE_SIZE - 1) / sizeof(mdbx_atomic_uint64_t)];
};

/*----------------------------------------------------------------------------*/

/* pseudo-error code, not exposed outside libmdbx */
#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 33)

/* Number of slots in the reader table.
 * This value was chosen somewhat arbitrarily. The 61 is a prime number,
 * and such readers plus a couple mutexes fit into single 4KB page.
 * Applications should set the table size using mdbx_env_set_maxreaders(). */
#define DEFAULT_READERS 61

enum db_flags {
  DB_PERSISTENT_FLAGS =
      MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP,

  /* mdbx_dbi_open() flags */
  DB_USABLE_FLAGS = DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE,

  DB_VALID = 0x80u /* DB handle is valid, for dbs_flags */,
  DB_POISON = 0x7fu /* update pending */,
  DB_INTERNAL_FLAGS = DB_VALID
};

#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS
MDBX_MAYBE_UNUSED static void static_checks(void) {
  STATIC_ASSERT(MDBX_WORDBITS == sizeof(void *) * CHAR_BIT);
  STATIC_ASSERT(UINT64_C(0x80000000) == (uint32_t)ENV_FATAL_ERROR);
  STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, "Oops, MDBX_MAX_DBI or CORE_DBS?");
  STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
                        ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
                    "Oops, some flags overlapped or wrong");
  STATIC_ASSERT_MSG((DB_INTERNAL_FLAGS & DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong");
  STATIC_ASSERT_MSG((DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong");
  STATIC_ASSERT(DB_PERSISTENT_FLAGS <= UINT8_MAX);
  STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong");

  STATIC_ASSERT_MSG((txn_state_flags & (txn_rw_begin_flags | txn_ro_begin_flags)) == 0,
                    "Oops, some txn flags overlapped or wrong");
  STATIC_ASSERT_MSG(((txn_rw_begin_flags | txn_ro_begin_flags | txn_state_flags) & txn_shrink_allowed) == 0,
                    "Oops, some txn flags overlapped or wrong");

  STATIC_ASSERT(sizeof(reader_slot_t) == 32);
#if MDBX_LOCKING > 0
  STATIC_ASSERT(offsetof(lck_t, wrt_lock) % MDBX_CACHELINE_SIZE == 0);
  STATIC_ASSERT(offsetof(lck_t, rdt_lock) % MDBX_CACHELINE_SIZE == 0);
#else
  STATIC_ASSERT(offsetof(lck_t, cached_oldest_txnid) % MDBX_CACHELINE_SIZE == 0);
  STATIC_ASSERT(offsetof(lck_t, rdt_length) % MDBX_CACHELINE_SIZE == 0);
#endif /* MDBX_LOCKING */
#if FLEXIBLE_ARRAY_MEMBERS
  STATIC_ASSERT(offsetof(lck_t, rdt) % MDBX_CACHELINE_SIZE == 0);
#endif /* FLEXIBLE_ARRAY_MEMBERS */

#if FLEXIBLE_ARRAY_MEMBERS
  STATIC_ASSERT(NODESIZE == offsetof(node_t, payload));
  STATIC_ASSERT(PAGEHDRSZ == offsetof(page_t, entries));
#endif /* FLEXIBLE_ARRAY_MEMBERS */
  STATIC_ASSERT(sizeof(clc_t) == 7 * sizeof(void *));
  STATIC_ASSERT(sizeof(kvx_t) == 16 * sizeof(void *));

#define KVX_SIZE_LN2 (MDBX_WORDBITS_LN2 + 1)
  STATIC_ASSERT(sizeof(kvx_t) == (1u << KVX_SIZE_LN2));
}
#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */

/******************************************************************************/

#ifndef __cplusplus

/* Internal prototypes */

MDBX_INTERNAL int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...);
MDBX_INTERNAL void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...);

/* audit.c */
MDBX_INTERNAL int audit_ex(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc);

/* mvcc-readers.c */
MDBX_INTERNAL bsr_t mvcc_bind_slot(MDBX_env *env);
MDBX_MAYBE_UNUSED MDBX_INTERNAL pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest);
MDBX_INTERNAL pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page);
MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rlocked, int *dead);

struct gc_reclaiming_obstacle;
MDBX_INTERNAL bool mvcc_kick_laggards(MDBX_txn *txn, const txnid_t laggard,
                                      struct gc_reclaiming_obstacle *optional_obstacle);

typedef struct rw_oldest_readed_shapshot_into {
  txnid_t oldest_txnid, steady_txnid;
} orsi_rw_t;
MDBX_INTERNAL orsi_rw_t mvcc_shapshot_oldest_rw(const MDBX_txn *const txn);

typedef struct ro_oldest_readed_shapshot_into {
  txnid_t oldest_txnid, thisprocess_oldest_txnid;
  txnid_t recent_txnid;
  size_t nreaders;
} orsi_ro_t;
MDBX_INTERNAL orsi_ro_t mvcc_shapshot_oldest_ro(const MDBX_txn *const txn, const bool need_thisprocess_oldest);

/* dxb.c */
MDBX_INTERNAL int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bits);
MDBX_INTERNAL int __must_check_result dxb_read_header(MDBX_env *env, meta_t *meta, const int lck_exclusive,
                                                      const mdbx_mode_t mode_bits);
enum resize_mode { implicit_grow, impilict_shrink, explicit_resize };
MDBX_INTERNAL int __must_check_result dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t size_pgno,
                                                 pgno_t limit_pgno, const enum resize_mode mode);
MDBX_INTERNAL int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, const bool enable, const bool force_whole);
MDBX_INTERNAL int dxb_msync(const MDBX_env *env, size_t length_pages, enum osal_syncmode_bits mode_bits);
MDBX_INTERNAL int dxb_fsync(const MDBX_env *env, enum osal_syncmode_bits mode_bits);
MDBX_INTERNAL int __must_check_result dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending,
                                                      troika_t *const troika);
#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
MDBX_INTERNAL void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn);
#else
MDBX_MAYBE_UNUSED static inline void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) {
  (void)env;
  (void)txn;
}
#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */

struct commit_timestamp {
  uint64_t start, prep, gc, audit, write, sync, gc_cpu;
};
MDBX_INTERNAL pgop_stat_t *txn_latency_gcprof(const MDBX_env *env, MDBX_commit_latency *latency);

MDBX_INTERNAL bool txn_refund(MDBX_txn *txn);
MDBX_INTERNAL bool txn_gc_detent(const MDBX_txn *const txn);
MDBX_INTERNAL int txn_check_badbits_parked(const MDBX_txn *txn, int bad_bits);
MDBX_INTERNAL void txn_done_cursors(MDBX_txn *txn);
MDBX_INTERNAL int txn_shadow_cursors(const MDBX_txn *parent, const size_t dbi);

MDBX_INTERNAL MDBX_txn *txn_alloc(const unsigned flags, MDBX_env *env);
MDBX_INTERNAL int txn_abort(MDBX_txn *txn, MDBX_commit_latency *latency);
MDBX_INTERNAL int txn_commit(MDBX_txn *txn, MDBX_commit_latency *latency, struct commit_timestamp *ts);
#if !(defined(_WIN32) || defined(_WIN64))
MDBX_INTERNAL void txn_abort_after_resurrect(MDBX_txn *txn);
#endif /* Windows */
MDBX_INTERNAL int txn_setup_primal(MDBX_txn *txn);

MDBX_INTERNAL int txn_nested_create(MDBX_txn *parent, bool readonly);
MDBX_INTERNAL int txn_nested_abort(MDBX_txn *txn);
MDBX_INTERNAL int txn_nested_commit(MDBX_txn *txn, struct commit_timestamp *ts);
MDBX_INTERNAL int txn_nested_checkpoint(MDBX_txn *txn, struct commit_timestamp *ts);
MDBX_INTERNAL int txn_nested_rollback(MDBX_txn *txn);
MDBX_INTERNAL MDBX_txn *txn_nested_fakero_begin(MDBX_txn *parent);
MDBX_INTERNAL int txn_nested_fakero_end(MDBX_txn *txn);

MDBX_INTERNAL MDBX_txn *txn_basal_create(const size_t max_dbi);
MDBX_INTERNAL void txn_basal_destroy(MDBX_txn *txn);
MDBX_INTERNAL int txn_basal_start(MDBX_txn *txn, unsigned flags);
MDBX_INTERNAL int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts);
MDBX_INTERNAL int txn_basal_end(MDBX_txn *txn, bool unlock);
MDBX_INTERNAL int txn_basal_checkpoint(MDBX_txn *txn, MDBX_txn_flags_t weakening_durability,
                                       struct commit_timestamp *ts);
MDBX_INTERNAL int txn_basal_rollback(MDBX_txn *txn);
MDBX_INTERNAL int txn_basal_update_tbl_roots(MDBX_txn *txn);

MDBX_INTERNAL int txn_ro_park(MDBX_txn *txn, bool autounpark);
MDBX_INTERNAL int txn_ro_unpark(MDBX_txn *txn);
MDBX_INTERNAL int txn_ro_start(MDBX_txn *txn, bool prepare_only);
MDBX_INTERNAL int txn_ro_clone(const MDBX_txn *const source, MDBX_txn *const clone);
MDBX_INTERNAL int txn_ro_reset(MDBX_txn *txn);
MDBX_INTERNAL void txn_ro_free(MDBX_txn *txn);

MDBX_INTERNAL int env_open(MDBX_env *env, mdbx_mode_t mode);
MDBX_INTERNAL int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, troika_t *troika);
MDBX_INTERNAL int env_sync(MDBX_env *env, bool force, bool nonblock);
MDBX_INTERNAL int env_close(MDBX_env *env, bool resurrect_after_fork);
MDBX_INTERNAL MDBX_txn *env_owned_wrtxn(const MDBX_env *env);
MDBX_INTERNAL int __must_check_result env_page_auxbuffer(MDBX_env *env);
MDBX_INTERNAL unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize);
MDBX_INTERNAL bool env_is_page_incore(MDBX_env *const env, pgno_t pgno);
MDBX_INTERNAL void env_clear_incore_cache(const MDBX_env *const env);

MDBX_INTERNAL void env_options_init(MDBX_env *env);
MDBX_INTERNAL void env_options_adjust_defaults(MDBX_env *env);
MDBX_INTERNAL void env_options_adjust_dp_limit(MDBX_env *env);
MDBX_INTERNAL pgno_t default_dp_limit(const MDBX_env *env);

MDBX_INTERNAL int __must_check_result tree_deepen_edge(MDBX_cursor *mc, int flags);
MDBX_INTERNAL int tree_deepen_lowest(MDBX_cursor *mc);
MDBX_INTERNAL intptr_t tree_diff_level(const MDBX_cursor *left, const MDBX_cursor *right);
MDBX_INTERNAL size_t tree_search_branch_configure(const MDBX_cursor *mc, const MDBX_val *key);
MDBX_INTERNAL sfr_t tree_search_foliage_configure(MDBX_cursor *mc, const MDBX_val *key);

enum page_search_flags {
  Z_MODIFY = 1,
  Z_ROOTONLY = 2,
  Z_FIRST = 4,
  Z_LAST = 8,
};
MDBX_INTERNAL int __must_check_result tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags);

static inline size_t tree_search_branch(const MDBX_cursor *mc, const MDBX_val *key) {
  return mc->clc->k.search_branch(mc, key);
}

static inline sfr_t tree_search_foliage(MDBX_cursor *mc, const MDBX_val *key) {
  return mc->clc->k.search_foliage(mc, key);
}

#if MDBX_ENABLE_BUNCHES_REMOVAL
MDBX_INTERNAL int tree_cutoff_twig(MDBX_cursor *mc, const pgno_t pgno, size_t deep, txnid_t parent_txnid,
                                   const bool whole_tree);
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
MDBX_INTERNAL int tree_curoff_range(MDBX_cursor *begin, MDBX_cursor *end, bool end_including);
MDBX_INTERNAL int tree_drop(MDBX_cursor *mc);
MDBX_INTERNAL int __must_check_result tree_rebalance(MDBX_cursor *mc);
MDBX_INTERNAL int __must_check_result tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key);
MDBX_INTERNAL void recalculate_merge_thresholds(MDBX_env *env);
MDBX_INTERNAL void recalculate_subpage_thresholds(MDBX_env *env);

MDBX_INTERNAL int __must_check_result tbl_fetch(MDBX_txn *txn, MDBX_cursor *mc, size_t dbi, const MDBX_val *name,
                                                unsigned wanna_flags);
MDBX_INTERNAL int __must_check_result tbl_create(MDBX_txn *txn, MDBX_cursor *mc, size_t slot, const MDBX_val *name,
                                                 unsigned db_flags);
MDBX_INTERNAL int __must_check_result tbl_setup(const MDBX_env *env, volatile kvx_t *const kvx, const tree_t *const db);
MDBX_INTERNAL int __must_check_result tbl_refresh(MDBX_txn *txn, size_t dbi);
MDBX_INTERNAL int __must_check_result tbl_purge(MDBX_cursor *mc);
MDBX_INTERNAL int __must_check_result tbl_stat_summary(const MDBX_txn *txn, MDBX_stat *st);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL txnid_t tbl_root_txnid(const MDBX_txn *txn, const size_t dbi);

/* coherency.c */
MDBX_INTERNAL bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report);
MDBX_INTERNAL int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp);
MDBX_INTERNAL int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta,
                                          const intptr_t pgno, uint64_t *timestamp);
MDBX_INTERNAL int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env);

/* histogram.c */
#define HISTOGRAM_LE0 1
MDBX_INTERNAL void histogram_acc_ex(const size_t value, struct MDBX_chk_histogram *histogram, unsigned options);
MDBX_MAYBE_UNUSED static inline void histogram_acc(const size_t value, struct MDBX_chk_histogram *histogram) {
  histogram_acc_ex(value, histogram, 0);
}
MDBX_INTERNAL MDBX_chk_line_t *histogram_dist(MDBX_chk_line_t *line, const struct MDBX_chk_histogram *histogram,
                                              const char *prefix, const char *first, bool amount);
MDBX_INTERNAL MDBX_chk_line_t *histogram_print(MDBX_chk_scope_t *scope, MDBX_chk_line_t *line,
                                               const struct MDBX_chk_histogram *histogram, const char *prefix,
                                               const char *first, bool amount);

/* print.c */
MDBX_INTERNAL void chk_line_end(MDBX_chk_line_t *line);
MDBX_INTERNAL __must_check_result MDBX_chk_line_t *chk_line_begin(MDBX_chk_scope_t *const scope,
                                                                  enum MDBX_chk_severity severity);
MDBX_INTERNAL MDBX_chk_line_t *chk_line_feed(MDBX_chk_line_t *line);
MDBX_INTERNAL MDBX_chk_line_t *chk_flush(MDBX_chk_line_t *line);
MDBX_INTERNAL size_t chk_print_wanna(MDBX_chk_line_t *line, size_t need);
MDBX_INTERNAL MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, const char *str);
MDBX_INTERNAL MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, const char *fmt, va_list args);
MDBX_INTERNAL MDBX_chk_line_t *MDBX_PRINTF_ARGS(2, 3) chk_print(MDBX_chk_line_t *line, const char *fmt, ...);
MDBX_INTERNAL void chk_println_va(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity, const char *fmt,
                                  va_list args);
MDBX_INTERNAL void chk_println(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity, const char *fmt, ...);

/* valid flags for mdbx_node_add() */
#define NODE_ADD_FLAGS (N_DUP | N_TREE | MDBX_RESERVE | MDBX_APPEND)

/* Get the page number pointed to by a branch node */
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_pgno(const node_t *const __restrict node) {
  pgno_t pgno = UNALIGNED_PEEK_32(node, node_t, child_pgno);
  return pgno;
}

/* Set the page number in a branch node */
static inline void node_set_pgno(node_t *const __restrict node, pgno_t pgno) {
  ASSERT(pgno >= MIN_PAGENO && pgno <= MAX_PAGENO);
  UNALIGNED_POKE_32(node, node_t, child_pgno, (uint32_t)pgno);
}

/* Get the size of the data in a leaf node */
MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ds(const node_t *const __restrict node) {
  return UNALIGNED_PEEK_32(node, node_t, dsize);
}

/* Set the size of the data for a leaf node */
static inline void node_set_ds(node_t *const __restrict node, size_t size) {
  ASSERT(size < INT_MAX);
  UNALIGNED_POKE_32(node, node_t, dsize, (uint32_t)size);
}

/* The size of a key in a node */
MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ks(const node_t *const __restrict node) {
  return UNALIGNED_PEEK_16(node, node_t, ksize);
}

/* Set the size of the key for a leaf node */
static inline void node_set_ks(node_t *const __restrict node, size_t size) {
  ASSERT(size < INT16_MAX);
  UNALIGNED_POKE_16(node, node_t, ksize, (uint16_t)size);
}

MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t node_flags(const node_t *const __restrict node) {
  return UNALIGNED_PEEK_8(node, node_t, flags);
}

static inline void node_set_flags(node_t *const __restrict node, uint8_t flags) {
  UNALIGNED_POKE_8(node, node_t, flags, flags);
}

/* Address of the key for the node */
MDBX_NOTHROW_PURE_FUNCTION static inline void *node_key(const node_t *const __restrict node) {
  return ptr_disp(node, NODESIZE);
}

/* Address of the data for a node */
MDBX_NOTHROW_PURE_FUNCTION static inline void *node_data(const node_t *const __restrict node) {
  return ptr_disp(node_key(node), node_ks(node));
}

/* Size of a node in a leaf page with a given key and data.
 * This is node header plus key plus data size. */
MDBX_NOTHROW_CONST_FUNCTION static inline size_t node_size_len(const size_t key_len, const size_t value_len) {
  return NODESIZE + EVEN_CEIL(key_len + value_len);
}
MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_size(const MDBX_val *key, const MDBX_val *value) {
  return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0);
}

MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_largedata_pgno(const node_t *const __restrict node) {
  ASSERT(node_flags(node) & N_BIG);
  return peek_pgno(node_data(node));
}

MDBX_INTERNAL int __must_check_result node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data,
                                                        const page_t *mp);

static inline int __must_check_result node_read(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) {
  data->iov_len = node_ds(node);
  data->iov_base = node_data(node);
  if (likely(node_flags(node) != N_BIG)) {
#if 0
    /* This is an example of a code that checks out-of-bounds by an incorrect/bad/crafted node.
     * Such checks look useful, but they are unreasonable really:
     *  - an each such check catches some damage case of the structure, apparently in one of the hot execution paths,
     *    but LEAVES several dozen more aside;
     *  - the overhead increases slightly,
     *    but there is also NO CERTAINTY that all or most of the corruption cases will be solved;
     *  - libmdbx already has a fairly complete page content validation mode by MDBX_VALIDATION, which leads
     *    calling the page_check() for each page from all page_get_xxx() variants, excepts page_get_unchecked();
     *  = So, the MDBX_VALIDATION must be used to work with untrusted data,
     *    but such checks are not necessary for trusted data.
     *
     * Thus, libmdbx allows a user, if necessary, to enable control of the database structure at the cost of reduced
     * performance. On the other hand, such approach allows not to lose productivity unnecessarily.
     *
     * Related issues:
     *  - https://sourcecraft.dev/dqdkfa/libmdbx/issues/290
     *  - https://github.com/Mithril-mine/libmdbx/pull/306
     */
    const char *data_end = ptr_disp(data->iov_base, data->iov_len);
    const char *page_tail = (const char *)((intptr_t)mp | /* Using the OR operation to get the tail of a real page
                                                             in case here is a dupsort nested sub-page even. */
                                           (intptr_t)(mc->txn->env->ps - 1));
    if (!MDBX_DISABLE_VALIDATION && unlikely(data_end > page_tail))
      return bad_page(mp, "node-data (size %zu bytes) beyond the end of page", data->iov_len);
#endif /* code example */
    return MDBX_SUCCESS;
  }
  return node_read_bigdata(mc, node, data, mp);
}

/*----------------------------------------------------------------------------*/

MDBX_INTERNAL int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno);

MDBX_INTERNAL int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data,
                                                    unsigned flags);

MDBX_INTERNAL int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key);

MDBX_INTERNAL void node_del(MDBX_cursor *mc, size_t ksize);

MDBX_INTERNAL node_t *node_shrink(page_t *mp, size_t indx, node_t *node);

#if MDBX_ENABLE_DBI_SPARSE

MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn,
                                                                                           intptr_t bmi);

static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) {
  cASSERT0(txn, bmi != 0);
  STATIC_ASSERT(sizeof(bmi) >= sizeof(txn->dbi_sparse[0]));
#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl)
  if (sizeof(txn->dbi_sparse[0]) <= sizeof(int))
    return __builtin_ctz((int)bmi);
  if (sizeof(txn->dbi_sparse[0]) == sizeof(long))
    return __builtin_ctzl((long)bmi);
#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_ctzll)
  return __builtin_ctzll(bmi);
#endif /* have(long long) && long long == uint64_t */
#endif /* GNU C */

#if defined(_MSC_VER)
  unsigned long index;
  if (sizeof(txn->dbi_sparse[0]) > 4) {
#if defined(_M_AMD64) || defined(_M_ARM64) || defined(_M_X64)
    _BitScanForward64(&index, bmi);
    return index;
#else
    if (bmi > UINT32_MAX) {
      _BitScanForward(&index, (uint32_t)((uint64_t)bmi >> 32));
      return index;
    }
#endif
  }
  _BitScanForward(&index, (uint32_t)bmi);
  return index;
#endif /* MSVC */

  return dbi_bitmap_ctz_fallback(txn, bmi);
}

static inline bool dbi_foreach_step(const MDBX_txn *const txn, size_t *bitmap_item, size_t *dbi) {
  const size_t bitmap_chunk = CHAR_BIT * sizeof(txn->dbi_sparse[0]);
  if (*bitmap_item & 1) {
    *bitmap_item >>= 1;
    return txn->dbi_state[*dbi] != 0;
  }
  if (*bitmap_item) {
    size_t bitmap_skip = dbi_bitmap_ctz(txn, *bitmap_item);
    *bitmap_item >>= bitmap_skip;
    *dbi += bitmap_skip - 1;
  } else {
    *dbi = (*dbi - 1) | (bitmap_chunk - 1);
    *bitmap_item = txn->dbi_sparse[(1 + *dbi) / bitmap_chunk];
    if (*bitmap_item == 0)
      *dbi += bitmap_chunk;
  }
  return false;
}

/* LY: Макрос целенаправленно сделан с одним циклом, чтобы сохранить возможность
 * использования оператора break */
#define TXN_FOREACH_DBI_FROM(TXN, I, FROM)                                                                             \
  for (size_t bitmap_item = TXN->dbi_sparse[0] >> FROM, I = FROM; I < TXN->n_dbi; ++I)                                 \
    if (dbi_foreach_step(TXN, &bitmap_item, &I))

#else

#define TXN_FOREACH_DBI_FROM(TXN, I, FROM)                                                                             \
  for (size_t I = FROM; I < TXN->n_dbi; ++I)                                                                           \
    if (TXN->dbi_state[I])

#endif /* MDBX_ENABLE_DBI_SPARSE */

#define TXN_FOREACH_DBI_ALL(TXN, I) TXN_FOREACH_DBI_FROM(TXN, I, 0)
#define TXN_FOREACH_DBI_USER(TXN, I) TXN_FOREACH_DBI_FROM(TXN, I, CORE_DBS)

MDBX_INTERNAL int dbi_import(MDBX_txn *txn, const size_t dbi);
MDBX_INTERNAL int dbi_gone(MDBX_txn *txn, const size_t dbi, const int rc);

struct dbi_snap_result {
  uint32_t sequence;
  unsigned flags;
};
MDBX_INTERNAL struct dbi_snap_result dbi_snap(const MDBX_env *env, const size_t dbi);

MDBX_INTERNAL int dbi_update(MDBX_txn *txn, bool keep);

static inline uint8_t dbi_state(const MDBX_txn *txn, const size_t dbi) {
  STATIC_ASSERT((int)DBI_DIRTY == MDBX_DBI_DIRTY && (int)DBI_STALE == MDBX_DBI_STALE &&
                (int)DBI_FRESH == MDBX_DBI_FRESH && (int)DBI_CREAT == MDBX_DBI_CREAT);

#if MDBX_ENABLE_DBI_SPARSE
  const size_t bitmap_chunk = CHAR_BIT * sizeof(txn->dbi_sparse[0]);
  const size_t bitmap_indx = dbi / bitmap_chunk;
  const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk;
  return likely(dbi < txn->n_dbi && (txn->dbi_sparse[bitmap_indx] & bitmap_mask) != 0) ? txn->dbi_state[dbi] : 0;
#else
  return likely(dbi < txn->n_dbi) ? txn->dbi_state[dbi] : 0;
#endif /* MDBX_ENABLE_DBI_SPARSE */
}

static inline bool dbi_changed(const MDBX_txn *txn, const size_t dbi) {
  const MDBX_env *const env = txn->env;
  eASSERT0(env, dbi_state(txn, dbi) & DBI_LINDO);
  const uint32_t snap_seq = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease);
  return unlikely(snap_seq != txn->dbi_seqs[dbi]);
}

static inline int dbi_check(const MDBX_txn *txn, const size_t dbi) {
  const uint8_t state = dbi_state(txn, dbi);
  if (likely((state & DBI_LINDO) != 0 && !dbi_changed(txn, dbi)))
    return (state & DBI_VALID) ? MDBX_SUCCESS : MDBX_BAD_DBI;

  /* Медленный путь: ленивая до-инициализацяи и импорт */
  return dbi_import((MDBX_txn *)txn, dbi);
}

static inline uint32_t dbi_seq_next(const MDBX_env *const env, size_t dbi) {
  uint32_t v = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease) + 1;
  return v ? v : 1;
}

MDBX_INTERNAL int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi,
                           MDBX_cmp_func keycmp, MDBX_cmp_func datacmp);

MDBX_INTERNAL int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func keycmp,
                           MDBX_cmp_func datacmp);

typedef struct defer_free_item {
  struct defer_free_item *next;
  uint64_t timestamp;
} defer_free_item_t;

MDBX_INTERNAL int dbi_defer_release(MDBX_env *const env, defer_free_item_t *const chain);
MDBX_INTERNAL int dbi_close_release(MDBX_env *env, MDBX_dbi dbi);
MDBX_INTERNAL const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback);

struct dbi_rename_result {
  defer_free_item_t *defer;
  int err;
};

MDBX_INTERNAL struct dbi_rename_result dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name);

MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL pgno_t pv2pages(uint16_t pv);

MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint16_t pages2pv(size_t pages);

MDBX_MAYBE_UNUSED MDBX_INTERNAL bool pv2pages_verify(void);

/*------------------------------------------------------------------------------
 * Nodes, Keys & Values length limitation factors:
 *
 * BRANCH_NODE_MAX
 *   Branch-page must contain at least two nodes, within each a key and a child
 *   page number. But page can't be split if it contains less that 4 keys,
 *   i.e. a page should not overflow before adding the fourth key. Therefore,
 *   at least 3 branch-node should fit in the single branch-page. Further, the
 *   first node of a branch-page doesn't contain a key, i.e. the first node
 *   is always require space just for itself. Thus:
 *       PAGESPACE = pagesize - page_hdr_len;
 *       BRANCH_NODE_MAX = even_floor(
 *         (PAGESPACE - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t));
 *       KEYLEN_MAX = BRANCH_NODE_MAX - node_hdr_len;
 *
 * LEAF_NODE_MAX
 *   Leaf-node must fit into single leaf-page, where a value could be placed on
 *   a large/overflow page. However, may require to insert a nearly page-sized
 *   node between two large nodes are already fill-up a page. In this case the
 *   page must be split to two if some pair of nodes fits on one page, or
 *   otherwise the page should be split to the THREE with a single node
 *   per each of ones. Such 1-into-3 page splitting is costly and complex since
 *   requires TWO insertion into the parent page, that could lead to split it
 *   and so on up to the root. Therefore double-splitting is avoided here and
 *   the maximum node size is half of a leaf page space:
 *       LEAF_NODE_MAX = even_floor(PAGESPACE / 2 - sizeof(indx_t));
 *       DATALEN_NO_OVERFLOW = LEAF_NODE_MAX - NODESIZE - KEYLEN_MAX;
 *
 *  - Table-node must fit into one leaf-page:
 *       TABLE_NAME_MAX = LEAF_NODE_MAX - node_hdr_len - sizeof(tree_t);
 *
 *  - Dupsort values itself are a keys in a dupsort-table and couldn't be longer
 *    than the KEYLEN_MAX. But dupsort node must not great than LEAF_NODE_MAX,
 *    since dupsort value couldn't be placed on a large/overflow page:
 *       DUPSORT_DATALEN_MAX = min(KEYLEN_MAX,
 *                                 max(DATALEN_NO_OVERFLOW, sizeof(tree_t));
 */

#define PAGESPACE(pagesize) ((pagesize) - PAGEHDRSZ)

#define BRANCH_NODE_MAX(pagesize)                                                                                      \
  (EVEN_FLOOR((PAGESPACE(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t)))

#define LEAF_NODE_MAX(pagesize) (EVEN_FLOOR(PAGESPACE(pagesize) / 2) - sizeof(indx_t))

#define MAX_GC1OVPAGE(pagesize) (PAGESPACE(pagesize) / sizeof(pgno_t) - 1)

MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_max(size_t pagesize, MDBX_db_flags_t flags) {
  ASSERT(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize));
  STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE >= 8);
  if (flags & MDBX_INTEGERKEY)
    return 8 /* sizeof(uint64_t) */;

  const intptr_t max_branch_key = BRANCH_NODE_MAX(pagesize) - NODESIZE;
  STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE -
                    /* sizeof(uint64) as a key */ 8 >
                sizeof(tree_t));
  if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
    const intptr_t max_dupsort_leaf_key = LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(tree_t);
    return (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key;
  }
  return max_branch_key;
}

MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
  size_t size_max;
  if (flags & MDBX_INTEGERKEY)
    size_max = 8 /* sizeof(uint64_t) */;
  else {
    const intptr_t max_branch_key = env->branch_nodemax - NODESIZE;
    STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE -
                      /* sizeof(uint64) as a key */ 8 >
                  sizeof(tree_t));
    if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
      const intptr_t max_dupsort_leaf_key = env->leaf_nodemax - NODESIZE - sizeof(tree_t);
      size_max = (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key;
    } else
      size_max = max_branch_key;
  }
  eASSERT0(env, size_max == keysize_max(env->ps, flags));
  return size_max;
}

MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_min(MDBX_db_flags_t flags) {
  return (flags & MDBX_INTEGERKEY) ? 4 /* sizeof(uint32_t) */ : 0;
}

MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_min(MDBX_db_flags_t flags) {
  if (flags & MDBX_INTEGERDUP)
    return 4 /* sizeof(uint32_t) */;
  else if (flags & MDBX_DUPFIXED)
    return sizeof(indx_t);
  else
    return 0;
}

MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_max(size_t pagesize, MDBX_db_flags_t flags) {
  ASSERT(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize));

  if (flags & MDBX_INTEGERDUP)
    return 8 /* sizeof(uint64_t) */;

  if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP))
    return keysize_max(pagesize, 0);

  const unsigned page_ln2 = log2n_powerof2(pagesize);
  const size_t hard = 0x7FF00000ul;
  const size_t hard_pages = hard >> page_ln2;
  STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO);
  const size_t pages_limit = PAGELIST_LIMIT / 4;
  const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2);
  return (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2;
}

MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
  size_t size_max;
  if (flags & MDBX_INTEGERDUP)
    size_max = 8 /* sizeof(uint64_t) */;
  else if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP))
    size_max = env_keysize_max(env, 0);
  else {
    const size_t hard = 0x7FF00000ul;
    const size_t hard_pages = hard >> env->ps2ln;
    STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO);
    const size_t pages_limit = PAGELIST_LIMIT / 4;
    const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << env->ps2ln);
    size_max = (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2;
  }
  eASSERT0(env, size_max == valsize_max(env->ps, flags));
  return size_max;
}

/*----------------------------------------------------------------------------*/

MDBX_NOTHROW_PURE_FUNCTION static inline size_t leaf_size(const MDBX_env *env, const MDBX_val *key,
                                                          const MDBX_val *data) {
  size_t node_bytes = node_size(key, data);
  if (node_bytes > env->leaf_nodemax)
    /* put on large/overflow page */
    node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t);

  return node_bytes + sizeof(indx_t);
}

MDBX_NOTHROW_PURE_FUNCTION static inline size_t branch_size(const MDBX_env *env, const MDBX_val *key) {
  /* Size of a node in a branch page with a given key.
   * This is just the node header plus the key, there is no data. */
  size_t node_bytes = node_size(key, nullptr);
  if (unlikely(node_bytes > env->branch_nodemax)) {
    /* put on large/overflow page, not implemented */
    panic_fmt(env, "node_size(key) %zu > %u branch_nodemax", node_bytes, env->branch_nodemax);
    node_bytes = node_size(key, nullptr) + sizeof(pgno_t);
  }

  return node_bytes + sizeof(indx_t);
}

MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t flags_db2sub(uint16_t db_flags) {
  uint16_t sub_flags = db_flags & MDBX_DUPFIXED;

  /* MDBX_INTEGERDUP => MDBX_INTEGERKEY */
#define SHIFT_INTEGERDUP_TO_INTEGERKEY 2
  STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) == MDBX_INTEGERKEY);
  sub_flags |= (db_flags & MDBX_INTEGERDUP) >> SHIFT_INTEGERDUP_TO_INTEGERKEY;

  /* MDBX_REVERSEDUP => MDBX_REVERSEKEY */
#define SHIFT_REVERSEDUP_TO_REVERSEKEY 5
  STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) == MDBX_REVERSEKEY);
  sub_flags |= (db_flags & MDBX_REVERSEDUP) >> SHIFT_REVERSEDUP_TO_REVERSEKEY;

  return sub_flags;
}

static inline bool check_table_flags(unsigned flags) {
  switch (flags & ~(MDBX_REVERSEKEY | MDBX_INTEGERKEY)) {
  default:
    NOTICE("invalid db-flags 0x%x", flags);
    return false;
  case MDBX_DUPSORT:
  case MDBX_DUPSORT | MDBX_REVERSEDUP:
  case MDBX_DUPSORT | MDBX_DUPFIXED:
  case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP:
  case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP:
  case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
  case MDBX_DB_DEFAULTS:
    return (flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) != (MDBX_REVERSEKEY | MDBX_INTEGERKEY);
  }
}

MDBX_MAYBE_UNUSED static inline int tbl_setup_ifneed(const MDBX_env *env, volatile kvx_t *const kvx,
                                                     const tree_t *const db) {
  return likely(kvx->clc.v.lmax) ? MDBX_SUCCESS : tbl_setup(env, kvx, db);
}

MDBX_MAYBE_UNUSED static inline int tbl_refresh_absent2baddbi(MDBX_txn *txn, size_t dbi) {
  int rc = tbl_refresh(txn, dbi);
  return likely(rc != MDBX_NOTFOUND) ? rc : MDBX_BAD_DBI;
}

/*----------------------------------------------------------------------------*/

MDBX_NOTHROW_PURE_FUNCTION static inline size_t pgno2bytes(const MDBX_env *env, size_t pgno) {
  eASSERT0(env, (1u << env->ps2ln) == env->ps);
  return pgno << env->ps2ln;
}

MDBX_NOTHROW_PURE_FUNCTION static inline page_t *pgno2page(const MDBX_env *env, size_t pgno) {
  return ptr_disp(env->dxb_mmap.base, pgno2bytes(env, pgno));
}

MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t bytes2pgno(const MDBX_env *env, size_t bytes) {
  eASSERT0(env, (env->ps >> env->ps2ln) == 1);
  return (pgno_t)(bytes >> env->ps2ln);
}

/* align to system page size */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t bytes_ceil2sp_bytes(const MDBX_env *env, size_t bytes);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t bytes_ceil2sp_pgno(const MDBX_env *env, size_t bytes);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t pgno_ceil2sp_bytes(const MDBX_env *env, size_t pgno);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL pgno_t pgno_ceil2sp_pgno(const MDBX_env *env, size_t pgno);

/* align to system allocation granularityor page size (MDBX_ROUNDING_TO_ALLOCATION_GRANULARITY) */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t bytes_ceil2os_bytes(const MDBX_env *env, size_t bytes);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t bytes_ceil2os_pgno(const MDBX_env *env, size_t bytes);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t pgno_ceil2os_bytes(const MDBX_env *env, size_t pgno);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL pgno_t pgno_ceil2os_pgno(const MDBX_env *env, size_t pgno);

MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t largechunk_npages(const MDBX_env *env, size_t bytes) {
  return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1;
}

typedef struct alignkey {
  MDBX_val key;
  uint64_t intbuf;
} alignkey_t;

static inline int check_key(const MDBX_cursor *mc, const MDBX_val *key, alignkey_t *alignkey) {
  /* coverity[logical_vs_bitwise] */
  if (unlikely(key->iov_len < mc->clc->k.lmin ||
               (key->iov_len > mc->clc->k.lmax && (mc->clc->k.lmin == mc->clc->k.lmax))))
    return MDBX_BAD_VALSIZE;

  alignkey->key = *key;
  if (mc->tree->flags & MDBX_INTEGERKEY) {
    if (alignkey->key.iov_len == 8) {
      if (unlikely(7 & (uintptr_t)alignkey->key.iov_base))
        /* copy instead of return error to avoid break compatibility */
        alignkey->key.iov_base = bcopy_8(&alignkey->intbuf, alignkey->key.iov_base);
    } else if (alignkey->key.iov_len == 4) {
      if (unlikely(3 & (uintptr_t)alignkey->key.iov_base))
        /* copy instead of return error to avoid break compatibility */
        alignkey->key.iov_base = bcopy_4(&alignkey->intbuf, alignkey->key.iov_base);
    } else
      return MDBX_BAD_VALSIZE;
  }
  return MDBX_SUCCESS;
}

MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val get_key(const node_t *node) {
  MDBX_val key;
  key.iov_len = node_ks(node);
  key.iov_base = node_key(node);
  return key;
}

static inline void get_key_optional(const node_t *node, MDBX_val *keyptr /* __may_null */) {
  if (keyptr)
    *keyptr = get_key(node);
}

MDBX_NOTHROW_PURE_FUNCTION static inline void *page2payload(const page_t *mp) { return ptr_disp(mp, PAGEHDRSZ); }

MDBX_NOTHROW_PURE_FUNCTION static inline const page_t *payload2page(const void *data) {
  return container_of(data, page_t, entries);
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_MAYBE_UNUSED static inline const page_t *ptr2page(const MDBX_env *env,
                                                                                  const void *ptr) {
  eASSERT0(env,
           ptr_dist(ptr, env->dxb_mmap.base) >= 0 && (size_t)ptr_dist(ptr, env->dxb_mmap.base) < env->dxb_mmap.limit);
  const uintptr_t mask = env->ps - 1;
  return (page_t *)((uintptr_t)ptr & ~mask);
}

MDBX_NOTHROW_PURE_FUNCTION static inline meta_t *page_meta(page_t *mp) { return (meta_t *)page2payload(mp); }

MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_numkeys(const page_t *mp) {
  ASSERT(mp->lower <= mp->upper);
  return mp->lower >> 1;
}

MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_room(const page_t *mp) {
  ASSERT(mp->lower <= mp->upper);
  return mp->upper - mp->lower;
}

MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_space(const MDBX_env *env) {
  STATIC_ASSERT(PAGEHDRSZ % 2 == 0);
  return env->ps - PAGEHDRSZ;
}

MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_used(const MDBX_env *env, const page_t *mp) {
  return page_space(env) - page_room(mp);
}

/* The percentage of space used in the page, in a percents. */
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline unsigned page_fill_percentum_x10(const MDBX_env *env,
                                                                                            const page_t *mp) {
  const size_t space = page_space(env);
  return (unsigned)((page_used(env, mp) * 1000 + space / 2) / space);
}

MDBX_NOTHROW_PURE_FUNCTION static inline node_t *page_node(const page_t *mp, size_t i) {
  ASSERT(page_type_compat(mp) == P_LEAF || page_type(mp) == P_BRANCH);
  ASSERT(page_numkeys(mp) > i);
  ASSERT(mp->entries[i] % 2 == 0);
  return ptr_disp(mp, mp->entries[i] + PAGEHDRSZ);
}

MDBX_NOTHROW_PURE_FUNCTION static inline void *page_dupfix_ptr(const page_t *mp, size_t i, size_t keysize) {
  ASSERT(page_type_compat(mp) == (P_LEAF | P_DUPFIX) && i == (indx_t)i && mp->dupfix_ksize == keysize);
  (void)keysize;
  return ptr_disp(mp, PAGEHDRSZ + mp->dupfix_ksize * (indx_t)i);
}

MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val page_dupfix_key(const page_t *mp, size_t i, size_t keysize) {
  MDBX_val r;
  r.iov_base = page_dupfix_ptr(mp, i, keysize);
  r.iov_len = mp->dupfix_ksize;
  return r;
}

/*----------------------------------------------------------------------------*/

MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint32_unchecked(const size_t expected_alignment,
                                                                           const MDBX_val *a, const MDBX_val *b) {
  ASSERT(a->iov_len == 4 && b->iov_len == 4);
  return CMP2INT(unaligned_peek_u32(expected_alignment, a->iov_base),
                 unaligned_peek_u32(expected_alignment, b->iov_base));
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline int
cmp_uint32_unaligned_unchecked(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint32_unchecked(1, a, b);
}

MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint64_unchecked(const size_t expected_alignment,
                                                                           const MDBX_val *a, const MDBX_val *b) {
  ASSERT(a->iov_len == 8 && b->iov_len == 8);
  return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base),
                 unaligned_peek_u64(expected_alignment, b->iov_base));
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline int
cmp_uint64_unaligned_unchecked(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint64_unchecked(1, a, b);
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint_unaligned(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint32_unaligned(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint64_unaligned(const MDBX_val *a, const MDBX_val *b);

#if MDBX_UNALIGNED_OK < 2 || MDBX_CHECKING > 0
/* Compare two items pointing at 2-byte aligned unsigned int's. */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint_align2(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint32_align2(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint64_align2(const MDBX_val *a, const MDBX_val *b);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint32_align2_unchecked(const MDBX_val *a,
                                                                                                    const MDBX_val *b) {
  return cmp_uint32_unchecked(2, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint64_align2_unchecked(const MDBX_val *a,
                                                                                                    const MDBX_val *b) {
  return cmp_uint64_unchecked(2, a, b);
}
#else
#define cmp_uint_align2 cmp_uint_unaligned
#define cmp_uint32_align2 cmp_uint_unaligned
#define cmp_uint64_align2 cmp_uint_unaligned
#define cmp_uint32_align2_unchecked cmp_uint32_unaligned_unchecked
#define cmp_uint64_align2_unchecked cmp_uint64_unaligned_unchecked
#endif /* !MDBX_UNALIGNED_OK || debug */

#if MDBX_UNALIGNED_OK < 4 || MDBX_CHECKING > 0
/* Compare two items pointing at 4-byte aligned unsigned int's. */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint_align4(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint32_align4(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_uint64_align4(const MDBX_val *a, const MDBX_val *b);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint32_align4_unchecked(const MDBX_val *a,
                                                                                                    const MDBX_val *b) {
  return cmp_uint32_unchecked(4, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint64_align4_unchecked(const MDBX_val *a,
                                                                                                    const MDBX_val *b) {
  return cmp_uint64_unchecked(4, a, b);
}
#else
#define cmp_uint_align4 cmp_uint_unaligned
#define cmp_uint32_align4 cmp_uint32_unaligned
#define cmp_uint64_align4 cmp_uint64_unaligned
#define cmp_uint32_align4_unchecked cmp_uint32_unaligned_unchecked
#define cmp_uint64_align4_unchecked cmp_uint64_unaligned_unchecked
#endif /* !MDBX_UNALIGNED_OK || debug */

/* Compare two items lexically */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lexical(const MDBX_val *a, const MDBX_val *b);

/* Compare two items in reverse byte order */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_reverse(const MDBX_val *a, const MDBX_val *b);

/* Fast non-lexically comparator */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lenfast(const MDBX_val *a, const MDBX_val *b);

MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l);

MDBX_NOTHROW_PURE_FUNCTION static inline bool eq_fast(const MDBX_val *a, const MDBX_val *b) {
  return unlikely(a->iov_len == b->iov_len) && eq_fast_slowpath(a->iov_base, b->iov_base, a->iov_len);
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b);

MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b);

static inline MDBX_cmp_func builtin_keycmp(MDBX_db_flags_t flags) {
  return (flags & MDBX_REVERSEKEY) ? cmp_reverse : (flags & MDBX_INTEGERKEY) ? cmp_uint_align2 : cmp_lexical;
}

static inline MDBX_cmp_func builtin_datacmp(MDBX_db_flags_t flags) {
  return !(flags & MDBX_DUPSORT)
             ? cmp_lenfast
             : ((flags & MDBX_INTEGERDUP) ? cmp_uint_unaligned
                                          : ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical));
}

static inline void clc_reset_methods(volatile clc_t *clc) {
  clc->search_branch = tree_search_branch_configure;
  clc->search_foliage = tree_search_foliage_configure;
}

/*----------------------------------------------------------------------------*/

MDBX_INTERNAL uint32_t combine_durability_flags(const uint32_t a, const uint32_t b);

MDBX_CONST_FUNCTION static inline lck_t *lckless_stub(const MDBX_env *env) {
  uintptr_t stub = (uintptr_t)&env->lckless_placeholder;
  /* align to avoid false-positive alarm from UndefinedBehaviorSanitizer */
  stub = (stub + MDBX_CACHELINE_SIZE - 1) & ~(MDBX_CACHELINE_SIZE - 1);
  return (lck_t *)stub;
}

#if !(defined(_WIN32) || defined(_WIN64))
MDBX_CONST_FUNCTION static inline int ignore_enosys(int err) {
#ifdef ENOSYS
  if (err == ENOSYS)
    return MDBX_RESULT_TRUE;
#endif /* ENOSYS */
#ifdef ENOIMPL
  if (err == ENOIMPL)
    return MDBX_RESULT_TRUE;
#endif /* ENOIMPL */
#ifdef ENOTSUP
  if (err == ENOTSUP)
    return MDBX_RESULT_TRUE;
#endif /* ENOTSUP */
#ifdef ENOSUPP
  if (err == ENOSUPP)
    return MDBX_RESULT_TRUE;
#endif /* ENOSUPP */
#ifdef EOPNOTSUPP
  if (err == EOPNOTSUPP)
    return MDBX_RESULT_TRUE;
#endif /* EOPNOTSUPP */
  return err;
}

MDBX_MAYBE_UNUSED MDBX_CONST_FUNCTION static inline int ignore_enosys_and_eagain(int err) {
  return (err == EAGAIN) ? MDBX_RESULT_TRUE : ignore_enosys(err);
}

MDBX_MAYBE_UNUSED MDBX_CONST_FUNCTION static inline int ignore_enosys_and_einval(int err) {
  return (err == EINVAL) ? MDBX_RESULT_TRUE : ignore_enosys(err);
}

MDBX_MAYBE_UNUSED MDBX_CONST_FUNCTION static inline int ignore_enosys_and_eremote(int err) {
  return (err == MDBX_EREMOTE) ? MDBX_RESULT_TRUE : ignore_enosys(err);
}

#endif /* defined(_WIN32) || defined(_WIN64) */

static inline int check_env(const MDBX_env *env, const bool wanna_active) {
  if (unlikely(!env))
    return MDBX_EINVAL;

  if (unlikely(env->signature.weak != env_signature))
    return MDBX_EBADSIGN;

  if (unlikely(env->flags & ENV_FATAL_ERROR))
    return MDBX_PANIC;

  if (wanna_active) {
#if MDBX_ENV_CHECKPID
    if (unlikely(env->pid != osal_getpid()) && env->pid) {
      ((MDBX_env *)env)->flags |= ENV_FATAL_ERROR;
      return MDBX_PANIC;
    }
#endif /* MDBX_ENV_CHECKPID */
    if (unlikely((env->flags & ENV_ACTIVE) == 0))
      return MDBX_EPERM;
    eASSERT0(env, env->dxb_mmap.base != nullptr);
  }

  return MDBX_SUCCESS;
}

static __always_inline int check_txn_anythread(const MDBX_txn *txn, int bad_bits) {
  if (unlikely(!txn))
    return MDBX_EINVAL;

  if (unlikely(txn->signature != txn_signature))
    return MDBX_EBADSIGN;

  if (bad_bits) {
    if (unlikely(!txn->env->dxb_mmap.base))
      return MDBX_EPERM;

    if (unlikely(txn->flags & bad_bits)) {
      if ((bad_bits & txn_ro_both) && unlikely(txn->flags & txn_ro_both))
        return MDBX_EACCESS;
      if ((bad_bits & MDBX_TXN_PARKED) == 0)
        return MDBX_BAD_TXN;
      return txn_check_badbits_parked(txn, bad_bits);
    }
  }

  cASSERT0(txn, (txn->flags & MDBX_TXN_FINISHED) ||
                    (txn->flags & MDBX_NOSTICKYTHREADS) == (txn->env->flags & MDBX_NOSTICKYTHREADS));
  return MDBX_SUCCESS;
}

static __always_inline int check_txn(const MDBX_txn *txn, int bad_bits) {
  int err = check_txn_anythread(txn, bad_bits);
#if MDBX_TXN_CHECKOWNER
  if (err == MDBX_SUCCESS && (txn->flags & (MDBX_NOSTICKYTHREADS | MDBX_TXN_FINISHED)) != MDBX_NOSTICKYTHREADS &&
      !(bad_bits /* abort/reset/txn-break */ == 0 &&
        ((txn->flags & (txn_ro_flat | MDBX_TXN_FINISHED)) == (txn_ro_flat | MDBX_TXN_FINISHED))) &&
      unlikely(txn->owner != osal_thread_self()))
    err = txn->owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN;
#endif /* MDBX_TXN_CHECKOWNER */

  return err;
}

static inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) {
  return check_txn(txn, (bad_bits | txn_ro_both) & ~MDBX_TXN_PARKED);
}

MDBX_NOTHROW_CONST_FUNCTION static inline txnid_t txn_basis_snapshot(const MDBX_txn *txn) {
  STATIC_ASSERT(((txn_ro_flat >> ((xMDBX_TXNID_STEP == 2) ? 16 : 17)) & xMDBX_TXNID_STEP) == xMDBX_TXNID_STEP);
  const txnid_t committed_txnid =
      txn->txnid - xMDBX_TXNID_STEP + ((txn->flags >> ((xMDBX_TXNID_STEP == 2) ? 16 : 17)) & xMDBX_TXNID_STEP);
  cASSERT0(txn, committed_txnid == ((txn->flags & txn_ro_flat) ? txn->txnid : txn->txnid - xMDBX_TXNID_STEP));
  return committed_txnid;
}

/*----------------------------------------------------------------------------*/

MDBX_INTERNAL void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno,
                                const bool lock_not_release);

MDBX_INTERNAL void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes);

MDBX_INTERNAL void munlock_all(const MDBX_env *env);

/*----------------------------------------------------------------------------*/
/* Cache coherence and mmap invalidation */
#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
#error "The MDBX_CPU_WRITEBACK_INCOHERENT must be defined before"
#elif MDBX_CPU_WRITEBACK_INCOHERENT
#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier()
#else
#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier()
#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */

MDBX_MAYBE_UNUSED static inline void osal_flush_incoherent_mmap(const void *addr, size_t nbytes,
                                                                const intptr_t pagesize) {
#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
#error "The MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined before"
#elif MDBX_MMAP_INCOHERENT_FILE_WRITE
  char *const begin = (char *)(-pagesize & (intptr_t)addr);
  char *const end = (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
  int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
  ASSERT(err == 0);
#else
  (void)pagesize;
#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */

#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
#error "The MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined before"
#elif MDBX_MMAP_INCOHERENT_CPU_CACHE
#ifdef DCACHE
  /* MIPS has cache coherency issues.
   * Note: for any nbytes >= on-chip cache size, entire is flushed. */
  cacheflush((void *)addr, nbytes, DCACHE);
#else
#error "Oops, cacheflush() not available"
#endif /* DCACHE */
#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */

#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE
  (void)addr;
  (void)nbytes;
#endif
}

/* Состояние курсора.
 *
 * плохой/poor:
 *  - неустановленный курсор с незаполненым стеком;
 *  - следует пропускать во всех циклах отслеживания/корректировки
 *    позиций курсоров;
 *  - допускаются только операции предполагающие установку абсолютной позиции;
 *  - в остальных случаях возвращается ENODATA.
 *
 *    У таких курсоров top = -1 и flags < 0, что позволяет дешево проверять и
 *    пропускать такие курсоры в циклах отслеживания/корректировки по условию
 *    probe_cursor->top < this_cursor->top.
 *
 * пустой/hollow:
 *  - частично инициализированный курсор, но без доступной пользователю позиции,
 *    поэтому нельзя выполнить какую-либо операцию без абсолютного (не
 *    относительного) позиционирования;
 *  - ki[top] может быть некорректным, в том числе >= page_numkeys(pg[top]).
 *
 *    У таких курсоров top >= 0, но flags < 0 (есть флажок z_hollow).
 *
 * установленный/pointed:
 *  - полностью инициализированный курсор с конкретной позицией с данными;
 *  - можно прочитать текущую строку, удалить её, либо выполнить
 *    относительное перемещение;
 *  - может иметь флажки z_after_delete, z_eof_hard и z_eof_soft;
 *  - наличие z_eof_soft означает что курсор перемещен за пределы данных,
 *    поэтому нелья прочитать текущие данные, либо удалить их.
 *
 *    У таких курсоров top >= 0 и flags >= 0 (нет флажка z_hollow).
 *
 * наполненный данными/filled:
 *  - это установленный/pointed курсор без флагов z_eof_soft;
 *  - за курсором есть даные, возможны CRUD операции в текущей позиции.
 *
 *    У таких курсоров top >= 0 и (unsigned)flags < z_eof_soft.
 *
 * Изменения состояния.
 *
 *  - Сбрасывается состояние курсора посредством top_and_flags |= z_poor_mark,
 *    что равносильно top = -1 вместе с flags |= z_poor_mark;
 *  - При позиционировании курсора сначала устанавливается top, а flags
 *    только в самом конце при отсутстви ошибок.
 *  - Повторное позиционирование first/last может начинаться
 *    с установки/обнуления только top без сброса flags, что позволяет работать
 *    быстрому пути внутри tree_search().
 *
 *  - Заморочки с концом данных:
 *     - mdbx_cursor_get(NEXT) выполняет две операции (перемещение и чтение),
 *       поэтому перемещение на последнюю строку строку всегда успешно,
 *       а ошибка возвращается только при последующем next().
 *       Однако, из-за этой двойственности семантика ситуации возврата ошибки
 *       из mdbx_cursor_get(NEXT) допускает разночтение/неопределенность, ибо
 *       не понятно к чему относится ошибка:
 *        - Если к чтению данных, то курсор перемещен и стоит после последней
 *          строки. Соответственно, чтение в текущей позиции запрещено,
 *          а при выполнении prev() курсор вернется на последнюю строку;
 *        - Если же ошибка относится к перемещению, то курсор не перемещен и
 *          остается на последней строке. Соответственно, чтение в текущей
 *          позиции допустимо, а при выполнении prev() курсор встанет
 *          на пред-последнюю строку.
 *        - Пикантность в том, что пользователи (так или иначе) полагаются
 *          на оба варианта поведения, при этом конечно ожидают что после
 *          ошибки MDBX_NEXT функция mdbx_cursor_eof() будет возвращать true.
 *     - далее добавляется схожая ситуация с MDBX_GET_RANGE, MDBX_LOWERBOUND,
 *       MDBX_GET_BOTH_RANGE и MDBX_UPPERBOUND. Тут при неуспехе поиска курсор
 *       может/должен стоять после последней строки.
 *     - далее добавляется MDBX_LAST. Тут курсор должен стоять на последней
 *       строке и допускать чтение в текузщей позиции,
 *       но mdbx_cursor_eof() должен возвращать true.
 *
 *    Решение = делаем два флажка z_eof_soft и z_eof_hard:
 *     - Когда установлен только z_eof_soft,
 *       функция mdbx_cursor_eof() возвращает true, но допускается
 *       чтение данных в текущей позиции, а prev() передвигает курсор
 *       на пред-последнюю строку.
 *     - Когда установлен z_eof_hard, чтение данных в текущей позиции
 *       не допускается, и mdbx_cursor_eof() также возвращает true,
 *       а prev() устанавливает курсора на последюю строку. */
enum cursor_state {
  /* Это вложенный курсор для вложенного дерева/страницы и является
     inner-элементом struct cursor_couple. */
  z_inner = 0x01,

  /* Происходит подготовка к обновлению GC,
     поэтому можно брать страницы из GC даже для FREE_DBI. */
  z_gcu_preparation = 0x02,

  /* Курсор только-что создан, поэтому допускается авто-установка
     в начало/конец, вместо возврата ошибки. */
  z_fresh = 0x04,

  /* Предыдущей операцией было удаление, поэтому курсор уже физически указывает
     на следующий элемент и соответствующая операция перемещения должна
     игнорироваться. */
  z_after_delete = 0x08,

  /* */
  z_disable_tree_search_fastpath = 0x10,

  /* Курсор логически в конце данных, но физически на последней строке,
     ki[top] == page_numkeys(pg[top]) - 1 и может читать данные в текущей позиции. */
  z_eof_soft = 0x20,

  /* Курсор логически за концом данных, поэтому следующий переход "назад"
     должен игнорироваться и/или приводить к установке на последнюю строку.
     В текущем же состоянии нельзя делать CRUD операции. */
  z_eof_hard = 0x40,

  /* За курсором нет данных, логически его позиция не определена,
     нельзя делать CRUD операции в текущей позиции.
     Относительное перемещение запрещено. */
  z_hollow = -128 /* 0x80 */,

  /* Маски для сброса/установки состояния. */
  z_clear_mask = z_inner | z_gcu_preparation,
  z_poor_mark = z_eof_hard | z_hollow | z_disable_tree_search_fastpath,
  z_fresh_mark = z_poor_mark | z_fresh
};

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_inner(const MDBX_cursor *mc) {
  return (mc->flags & z_inner) != 0;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_poor(const MDBX_cursor *mc) {
  const bool r = mc->top < 0;
  cASSERT0(mc, r == (mc->top_and_flags < 0));
  if (r && mc->subcur)
    cASSERT0(mc, mc->subcur->cursor.flags < 0 && mc->subcur->cursor.top < 0);
  return r;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_pointed(const MDBX_cursor *mc) {
  const bool r = mc->top >= 0;
  cASSERT0(mc, r == (mc->top_and_flags >= 0));
  if (!r && mc->subcur)
    cASSERT0(mc, is_poor(&mc->subcur->cursor));
  return r;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_hollow(const MDBX_cursor *mc) {
  const bool r = mc->flags < 0;
  if (!r) {
    cASSERT0(mc, mc->top >= 0);
    cASSERT0(mc, (mc->flags & z_eof_hard) || mc->ki[mc->top] < page_numkeys(mc->pg[mc->top]));
  } else if (mc->subcur)
    cASSERT0(mc, is_poor(&mc->subcur->cursor) || (is_pointed(mc) && mc->subcur->cursor.flags < 0));
  return r;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_eof(const MDBX_cursor *mc) {
  const bool r = z_eof_soft <= (uint8_t)mc->flags;
  return r;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_filled(const MDBX_cursor *mc) {
  const bool r = z_eof_hard > (uint8_t)mc->flags;
  return r;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_filled(const MDBX_cursor *mc) {
  return mc->subcur && is_filled(&mc->subcur->cursor);
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_pointed(const MDBX_cursor *mc) {
  return mc->subcur && is_pointed(&mc->subcur->cursor);
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_hollow(const MDBX_cursor *mc) {
  const bool r = !mc->subcur || is_hollow(&mc->subcur->cursor);
#if MDBX_CHECKING > 0
  if (!r) {
    cASSERT0(mc, is_filled(mc));
    const page_t *mp = mc->pg[mc->top];
    const node_t *node = page_node(mp, mc->ki[mc->top]);
    cASSERT0(mc, node_flags(node) & N_DUP);
  }
#endif /* MDBX_CHECKING > 0 */
  return r;
}

MDBX_MAYBE_UNUSED static inline void inner_gone(MDBX_cursor *mc) {
  if (mc->subcur) {
    TRACE("reset inner cursor %p", __Wpedantic_format_voidptr(&mc->subcur->cursor));
    mc->subcur->nested_tree.root = 0;
    mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
  }
}

MDBX_MAYBE_UNUSED static inline void be_poor(MDBX_cursor *mc) {
  const bool inner = is_inner(mc);
  if (inner) {
    mc->tree->root = 0;
    mc->top_and_flags = z_inner | z_poor_mark;
  } else {
    mc->top_and_flags |= z_poor_mark;
    inner_gone(mc);
  }
  cASSERT0(mc, is_poor(mc) && !is_pointed(mc) && !is_filled(mc));
  cASSERT0(mc, inner == is_inner(mc));
}

MDBX_MAYBE_UNUSED static inline void be_filled(MDBX_cursor *mc) {
  cASSERT0(mc, mc->top >= 0);
  cASSERT0(mc, mc->ki[mc->top] < page_numkeys(mc->pg[mc->top]));
  const bool inner = is_inner(mc);
  mc->flags &= z_clear_mask;
  cASSERT0(mc, is_filled(mc));
  cASSERT0(mc, inner == is_inner(mc));
}

MDBX_MAYBE_UNUSED static inline bool is_related(const MDBX_cursor *base, const MDBX_cursor *scan) {
  cASSERT0(base, base->top >= 0);
  return base->top <= scan->top && base != scan;
}

/* Флаги контроля/проверки курсора. */
enum cursor_checking {
  z_branch = 0x01 /* same as P_BRANCH for check_leaf_type() */,
  z_leaf = 0x02 /* same as P_LEAF for check_leaf_type() */,
  z_largepage = 0x04 /* same as P_LARGE for check_leaf_type() */,
  z_updating = 0x08 /* update/rebalance pending */,
  z_ignord = 0x10 /* don't check keys ordering */,
  z_dupfix = 0x20 /* same as P_DUPFIX for check_leaf_type() */,
  z_retiring = 0x40 /* refs to child pages may be invalid */,
  z_pagecheck = 0x80 /* perform page checking, see MDBX_VALIDATION */
};

MDBX_INTERNAL int __must_check_result cursor_validate(const MDBX_cursor *mc);

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline size_t cursor_dbi(const MDBX_cursor *mc) {
  cASSERT0(mc, mc->txn->signature == txn_signature);
  size_t dbi = mc->dbi_state - mc->txn->dbi_state;
  cASSERT0(mc, dbi < mc->txn->env->n_dbi);
  return dbi;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_dbi_changed(const MDBX_cursor *mc) {
  return dbi_changed(mc->txn, cursor_dbi(mc));
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t *cursor_dbi_state(const MDBX_cursor *mc) {
  return mc->dbi_state;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_gc(const MDBX_cursor *mc) {
  return mc->dbi_state == mc->txn->dbi_state + FREE_DBI;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_main(const MDBX_cursor *mc) {
  return mc->dbi_state == mc->txn->dbi_state + MAIN_DBI;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_core(const MDBX_cursor *mc) {
  return mc->dbi_state < mc->txn->dbi_state + CORE_DBS;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_cursor *cursor_outer(MDBX_cursor *mc) {
  cASSERT0(mc, mc->flags & z_inner);
  return &container_of(mc, cursor_couple_t, inner.cursor)->outer;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline tree_t *cursor_outer_tree(MDBX_cursor *mc) {
  cASSERT0(mc, (mc->flags & z_inner) != 0);
  subcur_t *mx = container_of(mc->tree, subcur_t, nested_tree);
  cursor_couple_t *couple = container_of(mx, cursor_couple_t, inner);
  cASSERT0(mc, mc->tree == &couple->outer.subcur->nested_tree && &mc->clc->k == &couple->outer.clc->v);
  return couple->outer.tree;
}

MDBX_MAYBE_UNUSED static inline int cursor_dbi_dbg(const MDBX_cursor *mc) {
  /* Debugging output value of a cursor's DBI: Negative for a sub-cursor. */
  const int dbi = cursor_dbi(mc);
  return (mc->flags & z_inner) ? -dbi : dbi;
}

MDBX_MAYBE_UNUSED static inline int __must_check_result cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) {
  TRACE("pushing page %" PRIaPGNO " on db %d cursor %p", mp->pgno, cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc));
  if (unlikely(mc->top >= CURSOR_STACK_SIZE - 1)) {
    be_poor(mc);
    mc->txn->flags |= MDBX_TXN_ERROR;
    return MDBX_CURSOR_FULL;
  }
  mc->top += 1;
  mc->pg[mc->top] = mp;
  mc->ki[mc->top] = ki;
  return MDBX_SUCCESS;
}

MDBX_MAYBE_UNUSED static inline void cursor_pop(MDBX_cursor *mc) {
  TRACE("popped page %" PRIaPGNO " off db %d cursor %p", mc->pg[mc->top]->pgno, cursor_dbi_dbg(mc),
        __Wpedantic_format_voidptr(mc));
  cASSERT0(mc, mc->top >= 0);
  mc->top -= 1;
}

MDBX_NOTHROW_PURE_FUNCTION static inline bool check_leaf_type(const MDBX_cursor *mc, const page_t *mp) {
  return (((page_type(mp) ^ mc->checking) & (z_branch | z_leaf | z_largepage | z_dupfix)) == 0);
}

MDBX_INTERNAL int cursor_check(const MDBX_cursor *mc, int txn_bad_bits);

/* без необходимости доступа к данным, без активации припаркованных транзакций. */
static inline int cursor_check_pure(const MDBX_cursor *mc) {
  return cursor_check(mc, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
}

/* для чтения данных, с активацией припаркованных транзакций. */
static inline int cursor_check_ro(const MDBX_cursor *mc) { return cursor_check(mc, MDBX_TXN_BLOCKED); }

/* для записи данных. */
static inline int cursor_check_rw(const MDBX_cursor *mc) {
  return cursor_check(mc, (MDBX_TXN_BLOCKED - MDBX_TXN_PARKED) | txn_ro_both);
}

MDBX_INTERNAL MDBX_cursor *cursor_eot(MDBX_cursor *cursor, MDBX_txn *txn);
MDBX_INTERNAL int cursor_shadow(MDBX_cursor *cursor, MDBX_txn *nested, const size_t dbi);

MDBX_INTERNAL MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc, MDBX_cursor *cdst);
MDBX_INTERNAL MDBX_cursor *cursor_copy_position(const MDBX_cursor *csrc, MDBX_cursor *cdst);

MDBX_INTERNAL int __must_check_result cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
                                                 const MDBX_cursor_op op);

MDBX_INTERNAL int __must_check_result cursor_check_multiple(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
                                                            unsigned flags);

MDBX_INTERNAL int __must_check_result cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
                                                          unsigned flags);

MDBX_INTERNAL int __must_check_result cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags);

MDBX_INTERNAL int __must_check_result cursor_validate_updating(MDBX_cursor *mc);

MDBX_INTERNAL int __must_check_result cursor_del(MDBX_cursor *mc, unsigned flags);

MDBX_INTERNAL int __must_check_result cursor_sibling_left(MDBX_cursor *mc);
MDBX_INTERNAL int __must_check_result cursor_sibling_right(MDBX_cursor *mc);

typedef struct cursor_set_result {
  int err;
  bool exact;
} csr_t;

MDBX_INTERNAL csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op);

MDBX_INTERNAL int __must_check_result inner_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result inner_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
                                                  MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
                                                 MDBX_val *__restrict data);

MDBX_INTERNAL int __must_check_result inner_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result inner_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
                                                 MDBX_val *__restrict data, MDBX_cursor_op op);
MDBX_INTERNAL int __must_check_result outer_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
                                                 MDBX_val *__restrict data, MDBX_cursor_op op);

MDBX_INTERNAL int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree,
                                   kvx_t *const kvx);

MDBX_INTERNAL int __must_check_result cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi);

MDBX_INTERNAL int __must_check_result cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp);

MDBX_INTERNAL int __must_check_result cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data);

MDBX_INTERNAL MDBX_cursor *cursor_clone_slightly(const MDBX_cursor *csrc, cursor_couple_t *couple);
MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_cursor *cursor_clone_complete(const MDBX_cursor *csrc, cursor_couple_t *couple);

MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cursor_on_first(const MDBX_cursor *mc);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cursor_on_last(const MDBX_cursor *mc);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL intptr_t cursor_cmp(const MDBX_cursor *left, const MDBX_cursor *right);

MDBX_INTERNAL int cursor_distribute(const MDBX_cursor *begin, const MDBX_cursor *end, MDBX_cursor **array,
                                    size_t array_size, int level);
MDBX_INTERNAL int cursor_scroll_forward(MDBX_cursor *mc, intptr_t amount, int level);
MDBX_INTERNAL int cursor_scroll_backward(MDBX_cursor *mc, intptr_t amount, int level);

typedef struct cursor_distance_result {
  int err;
  size_t distance;
} cdr_t;
MDBX_INTERNAL cdr_t cursor_distance(MDBX_cursor *begin, const MDBX_cursor *end, int level);

/*----------------------------------------------------------------------------*/

/* Update sub-page pointer, if any, in mc->subcur.
 * Needed when the node which contains the sub-page may have moved.
 * Called with mp = mc->pg[mc->top], ki = mc->ki[mc->top]. */
MDBX_MAYBE_UNUSED static inline void cursor_inner_refresh(const MDBX_cursor *mc, const page_t *mp, unsigned ki) {
  cASSERT0(mc, is_leaf(mp));
  const node_t *node = page_node(mp, ki);
  if ((node_flags(node) & (N_DUP | N_TREE)) == N_DUP)
    mc->subcur->cursor.pg[0] = node_data(node);
}

MDBX_MAYBE_UNUSED MDBX_INTERNAL bool cursor_is_tracked(const MDBX_cursor *mc);

static inline void cursor_reset(cursor_couple_t *couple) {
  couple->outer.top_and_flags = z_fresh_mark;
  couple->inner.cursor.top_and_flags = z_fresh_mark | z_inner;
}

static inline void cursor_drown(cursor_couple_t *couple) {
  couple->outer.top_and_flags = z_poor_mark;
  couple->inner.cursor.top_and_flags = z_poor_mark | z_inner;
  couple->outer.txn = nullptr;
  couple->inner.cursor.txn = nullptr;
  couple->outer.tree = nullptr;
  /* сохраняем clc-указатель, так он используется для вычисления dbi в mdbx_cursor_renew(). */
  couple->outer.dbi_state = nullptr;
  couple->inner.cursor.dbi_state = nullptr;
}

static inline size_t dpl_setlen(dpl_t *dl, size_t len) {
  static const page_t dpl_stub_pageE = {INVALID_TXNID,
                                        0,
                                        P_BAD,
                                        {0},
                                        /* pgno */ ~(pgno_t)0};
  ASSERT(dpl_stub_pageE.flags == P_BAD && dpl_stub_pageE.pgno == P_INVALID);
  dl->length = len;
  dl->items[len + 1].ptr = (page_t *)&dpl_stub_pageE;
  dl->items[len + 1].pgno = P_INVALID;
#if MDBX_DPL_CACHE_NPAGES
  dl->items[len + 1].npages = 1;
#endif /* MDBX_DPL_CACHE_NPAGES */
  return len;
}

MDBX_INTERNAL int __must_check_result txn_dpl_alloc(MDBX_txn *txn);

MDBX_INTERNAL void txn_dpl_free(MDBX_txn *txn);

MDBX_INTERNAL dpl_t *txn_dpl_reserve(MDBX_txn *txn, size_t size);

MDBX_INTERNAL __noinline dpl_t *txn_dpl_sort_slowpath(const MDBX_txn *txn);

static inline dpl_t *txn_dpl_sort(const MDBX_txn *txn) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  dpl_t *dl = txn->wr.dirtylist;
  cASSERT0(txn, dl->length <= PAGELIST_LIMIT);
  cASSERT0(txn, dl->sorted <= dl->length);
  cASSERT0(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  return likely(dl->sorted == dl->length) ? dl : txn_dpl_sort_slowpath(txn);
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL __noinline size_t txn_dpl_search(const MDBX_txn *txn, pgno_t pgno);

MDBX_MAYBE_UNUSED MDBX_INTERNAL const page_t *debug_txn_dpl_find(const MDBX_txn *txn, const pgno_t pgno);

MDBX_NOTHROW_PURE_FUNCTION static inline unsigned dp_npages(const dp_t *dp) {
#if MDBX_DPL_CACHE_NPAGES
  ASSERT(likely((dp->ptr->flags & P_LARGE) == 0) ? 1 : dp->ptr->pages == dp->npages);
  return dp->npages;
#else
  return likely((dp->ptr->flags & P_LARGE) == 0) ? 1 : dp->ptr->pages;
#endif /* MDBX_DPL_CACHE_NPAGES */
}

MDBX_NOTHROW_PURE_FUNCTION static inline unsigned dpl_npages(const dpl_t *dl, size_t i) {
  ASSERT(0 <= (intptr_t)i && i <= dl->length);
  return dp_npages(dl->items + i);
}

MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t dpl_endpgno(const dpl_t *dl, size_t i) {
  return dpl_npages(dl, i) + dl->items[i].pgno;
}

MDBX_NOTHROW_PURE_FUNCTION static inline bool txn_dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  dpl_t *dl = txn->wr.dirtylist;
  cASSERT0(txn, dl->sorted == dl->length);
  cASSERT0(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  size_t const n = txn_dpl_search(txn, pgno);
  cASSERT0(txn, n >= 1 && n <= dl->length + 1);
  cASSERT0(txn, pgno <= dl->items[n].pgno);
  cASSERT0(txn, pgno > dl->items[n - 1].pgno);
  const bool rc =
      /* intersection with founded */ pgno + npages > dl->items[n].pgno ||
      /* intersection with prev */ dpl_endpgno(dl, n - 1) > pgno;
  if (CHECKS1_ENABLED()) {
    bool check = false;
    for (size_t i = 1; i <= dl->length; ++i) {
      const page_t *const dp = dl->items[i].ptr;
      if (!(dp->pgno /* begin */ >= /* end */ pgno + npages || dpl_endpgno(dl, i) /* end */ <= /* begin */ pgno)) {
        check = true;
        break;
      }
    }
    cASSERT0(txn, check == rc);
  }
  return rc;
}

MDBX_NOTHROW_PURE_FUNCTION static inline size_t txn_dpl_exist(const MDBX_txn *txn, pgno_t pgno) {
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
  dpl_t *dl = txn->wr.dirtylist;
  size_t i = txn_dpl_search(txn, pgno);
  cASSERT0(txn, (int)i > 0);
  return (dl->items[i].pgno == pgno) ? i : 0;
}

MDBX_INTERNAL void txn_dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages);

static inline void txn_dpl_remove(const MDBX_txn *txn, size_t i) {
  txn_dpl_remove_ex(txn, i, dpl_npages(txn->wr.dirtylist, i));
}

MDBX_INTERNAL int __must_check_result txn_dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages);

MDBX_MAYBE_UNUSED MDBX_INTERNAL bool txn_dpl_check(MDBX_txn *txn);

MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t txn_dpl_age(const MDBX_txn *txn, size_t i) {
  cASSERT0(txn, (txn->flags & (txn_ro_both | MDBX_WRITEMAP)) == 0);
  const dpl_t *dl = txn->wr.dirtylist;
  ASSERT((intptr_t)i > 0 && i <= dl->length);
  size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
  return txn->wr.dirtylru - (uint32_t)*ptr;
}

MDBX_INTERNAL void txn_dpl_lru_reduce(MDBX_txn *txn);

static inline uint32_t txn_dpl_lru_turn(MDBX_txn *txn) {
  txn->wr.dirtylru += 1;
  if (unlikely(txn->wr.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0)
    txn_dpl_lru_reduce(txn);
  return txn->wr.dirtylru;
}

MDBX_INTERNAL void txn_dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled);

MDBX_INTERNAL void txn_dpl_clear(MDBX_txn *txn);

typedef struct defrag_arc {
  pgno_t key_or_pgno;
  pgno_t parent;
  pgno_t mapped : 31;
  bool engaged : 1;
  pgno_t npages : 31;
  bool gc : 1;
} da_t;

typedef struct defrag_map_list {
  size_t length;
  size_t limit;
  da_t items[1];
} dml_t;

MDBX_MAYBE_UNUSED MDBX_INTERNAL dml_t *dml_alloc(size_t size);
MDBX_MAYBE_UNUSED MDBX_INTERNAL void dml_sort(dml_t *);
MDBX_MAYBE_UNUSED MDBX_INTERNAL da_t *dml_search(dml_t *, pgno_t key);
MDBX_MAYBE_UNUSED MDBX_INTERNAL da_t *dml_append(dml_t **pdml, pgno_t key);

MDBX_MAYBE_UNUSED static inline da_t *dml_search_exact(dml_t *dml, pgno_t pgno) {
  da_t *arc = dml_search(dml, pgno);
  return (arc < dml->items + dml->length && arc->key_or_pgno == pgno) ? arc : nullptr;
}

MDBX_MAYBE_UNUSED static inline void dml_free(dml_t **pdml) {
  if (likely(*pdml)) {
    osal_free(*pdml);
    *pdml = nullptr;
  }
}

struct gc_reclaiming_obstacle {
  mdbx_pid_t pid;
  mdbx_tid_t tid;
  txnid_t txnid;
};

/* Гистограмма решения нарезки фрагментов для ситуации нехватки идентификаторов/слотов. */
typedef struct gc_dense_histogram {
  /* Размер массива одновременно задаёт максимальный размер последовательностей,
   * с которыми решается задача распределения.
   *
   * Использование длинных последовательностей контрпродуктивно, так как такие последовательности будут
   * создавать/воспроизводить/повторять аналогичные затруднения при последующей переработке. Однако,
   * в редких ситуациях это может быть единственным выходом. */
  unsigned end;
  pgno_t array[31];
} gc_dense_histogram_t;

typedef struct gc_update_context {
  unsigned loop;
  unsigned goodchunk;
  bool dense;
  pgno_t prev_first_unallocated;
  size_t retired_stored;
  size_t return_reserved_lo, return_reserved_hi;
  txnid_t gc_first;
  intptr_t return_left;
#ifndef MDBX_DEBUG_GCU
#define MDBX_DEBUG_GCU 0
#endif
#if MDBX_DEBUG_GCU
  struct {
    txnid_t prev;
    unsigned n;
  } dbg;
#endif /* MDBX_DEBUG_GCU */
  rkl_t sequel;
#if MDBX_ENABLE_BIGFOOT
  txnid_t bigfoot;
#endif /* MDBX_ENABLE_BIGFOOT */
  union {
    MDBX_cursor cursor;
    cursor_couple_t couple;
  };
  gc_dense_histogram_t dense_histogram;
} gcu_t;

MDBX_INTERNAL int gc_put_init(MDBX_txn *txn, gcu_t *ctx);
MDBX_INTERNAL void gc_put_destroy(gcu_t *ctx);

#define ALLOC_DEFAULT 0     /* штатное/обычное выделение страниц */
#define ALLOC_UNIMPORTANT 1 /* запрос неважен, невозможность выделения не приведет к ошибке транзакции */
#define ALLOC_RESERVE 2     /* подготовка резерва для обновления GC, без аллокации */
#define ALLOC_COALESCE 4    /* внутреннее состояние/флажок */
#define ALLOC_SHOULD_SCAN 8 /* внутреннее состояние/флажок */
#define ALLOC_LIFO 16       /* внутреннее состояние/флажок */
#define ALLOC_EXACTLY 32

MDBX_INTERNAL pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags);

MDBX_INTERNAL pgr_t gc_alloc_single(const MDBX_cursor *const mc);
MDBX_INTERNAL int gc_update(MDBX_txn *txn, gcu_t *ctx);

MDBX_NOTHROW_PURE_FUNCTION static inline size_t gc_stockpile(const MDBX_txn *txn) {
  return pnl_size(txn->wr.repnl) + txn->wr.loose_count;
}

MDBX_NOTHROW_PURE_FUNCTION static inline size_t gc_chunk_bytes(const size_t chunk) {
  return (chunk + 1) * sizeof(pgno_t);
}

MDBX_INTERNAL bool gc_repnl_has_span(const MDBX_txn *txn, const size_t num);
MDBX_INTERNAL pgno_t gc_repnl_get_sequence(MDBX_txn *txn, const size_t num, uint8_t flags);
MDBX_INTERNAL pgno_t gc_repnl_get_single(MDBX_txn *txn);

static inline bool gc_is_reclaimed(const MDBX_txn *txn, const txnid_t id) {
  return rkl_contain(&txn->wr.gc.reclaimed, id) || rkl_contain(&txn->wr.gc.comeback, id);
}

MDBX_NOTHROW_PURE_FUNCTION static inline bool gc_may_clean_reclaimed(const MDBX_txn *txn) {
  return pnl_size(txn->wr.repnl) + txn->wr.loose_count + (txn->geo.end_pgno - txn->geo.first_unallocated) >
         txn->dbs[FREE_DBI].height * 2u + 3u;
}

static inline txnid_t txnid_min(txnid_t a, txnid_t b) { return (a < b) ? a : b; }

static inline txnid_t txnid_max(txnid_t a, txnid_t b) { return (a > b) ? a : b; }

MDBX_INTERNAL MDBX_cursor *gc_cursor_init(MDBX_txn *txn);
MDBX_INTERNAL int gc_merge_loose(MDBX_txn *txn);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL const char *gc_check_keylen(size_t const key_len);
MDBX_INTERNAL const char *gc_check_rowdata(const MDBX_txn *const txn, const MDBX_val data);

typedef struct gc_pnl_result {
  const_pnl_t pnl;
  int err;
  const char *reason;
} glr_t;
MDBX_INTERNAL glr_t gc_row_pnl(const MDBX_txn *const txn, const MDBX_val data);

typedef struct defract_context {
  MDBX_txn *txn;
  dml_t *arcs;
  pnl_t lp_reserve;
  pnl_t temp;
  pgno_t cycle_preprogress;
  pgno_t cycle_pages_scheduled;
  pgno_t cycle_pages_moved;
  pgno_t stumble_pgno, stumble_span, lp_backlog;
  pgno_t remapped_edge;
  pgno_t defrag_edge;
  pgno_t retreat_edge;
  pgno_t move_batch_size;
  size_t payload_items;
  size_t progress_counter;
  uint8_t wallclock_trottle;
  uint8_t stumble_retry;
  uint8_t stopping_reasons;
  unsigned cycle;
  pgno_t payload_pages;
  pgno_t largepage_max, largepage_amountleft, largepage_count;
  pgno_t summary_depth;
  void *user_ctx;
  MDBX_defrag_notify_func user_callback;
  size_t total_pages_moved;

  pgno_t walk_stack[32];
  pgno_t walk_cutoff;
  pgno_t notify_watchmask;

#if MDBX_CHECKING > 1
  pnl_t repnl_clone;
#endif /* MDBX_CHECKING > 1 */
  pgno_t defrag_atleast, defrag_enough, before_defrag, last_allocated;
  pgno_t gc_tree_pages, gc_retained_pages;
  uint64_t start_timestamp;
  uint64_t wallclock_atleast;
  uint64_t wallclock_detent;
  uint64_t cycle_initial_score;

  txnid_t stopor;
  struct gc_reclaiming_obstacle gc_obstacle;
  struct cache_item {
    pgno_t pgno;
    unsigned cost;
  } cache_cost[64];
} dfc_t;

MDBX_INTERNAL int defrag_init(dfc_t *dfc, MDBX_txn *txn, size_t defrag_atleast_pages,
                              size_t spend_atleast_wallclock_16dot16, size_t defrag_enough_pages,
                              size_t limit_spend_wallclock_16dot16, intptr_t preferred_move_batch_size);
MDBX_INTERNAL void defrag_destroy(dfc_t *dfc);
MDBX_INTERNAL int defrag_cycle(dfc_t *dfc);
MDBX_INTERNAL void defrag_milestone(dfc_t *dfc);

MDBX_MAYBE_UNUSED static inline bool defrag_discontinued(const dfc_t *dfc) {
  return dfc->stopping_reasons >= MDBX_defrag_discontinued;
}

MDBX_MAYBE_UNUSED static inline bool defrag_aborted(const dfc_t *dfc) {
  return dfc->stopping_reasons >= MDBX_defrag_aborted;
}

MDBX_INTERNAL uint64_t defrag_score(dfc_t *dfc, size_t allocated_pages);
MDBX_INTERNAL uint64_t defrag_result(dfc_t *dfc, MDBX_defrag_result_t *out, uint64_t now_cache);

MDBX_INTERNAL int lck_setup(MDBX_env *env, mdbx_mode_t mode);
#if MDBX_LOCKING > MDBX_LOCKING_SYSV
MDBX_INTERNAL int lck_ipclock_stubinit(osal_ipclock_t *ipc);
MDBX_INTERNAL int lck_ipclock_destroy(osal_ipclock_t *ipc);
#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */

MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag);

MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const mdbx_pid_t current_pid);

MDBX_INTERNAL int lck_seize(MDBX_env *env);

MDBX_INTERNAL int lck_downgrade(MDBX_env *env);

MDBX_MAYBE_UNUSED MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait);

MDBX_INTERNAL int lck_rdt_lock(MDBX_env *env);

MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env);

MDBX_INTERNAL int lck_txn_lock(MDBX_env *env, bool dont_wait);

MDBX_INTERNAL void lck_txn_unlock(MDBX_env *env);

MDBX_INTERNAL int lck_rpid_set(MDBX_env *env);

MDBX_INTERNAL int lck_rpid_clear(MDBX_env *env);

MDBX_INTERNAL int lck_rpid_check(MDBX_env *env, uint32_t pid);

static inline uint64_t meta_sign_calculate(const meta_t *meta) {
  uint64_t sign = DATASIGN_NONE;
#if 0 /* TODO */
  sign = hippeus_hash64(...);
#else
  (void)meta;
#endif
  /* LY: newer returns DATASIGN_NONE or DATASIGN_WEAK */
  return (sign > DATASIGN_WEAK) ? sign : ~sign;
}

static inline uint64_t meta_sign_get(const volatile meta_t *meta) { return unaligned_peek_u64_volatile(4, meta->sign); }

static inline void meta_sign_as_steady(meta_t *meta) { unaligned_poke_u64(4, meta->sign, meta_sign_calculate(meta)); }

static inline bool meta_is_steady(const volatile meta_t *meta) { return SIGN_IS_STEADY(meta_sign_get(meta)); }

MDBX_INTERNAL troika_t meta_tap(const MDBX_env *env);
MDBX_INTERNAL unsigned meta_eq_mask(const troika_t *troika);
MDBX_INTERNAL bool meta_should_retry(const MDBX_env *env, troika_t *troika);
MDBX_MAYBE_UNUSED MDBX_INTERNAL bool troika_verify_fsm(void);

struct meta_ptr {
  txnid_t txnid;
  union {
    const volatile meta_t *ptr_v;
    const meta_t *ptr_c;
  };
  size_t is_steady;
};

MDBX_INTERNAL meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n);
MDBX_INTERNAL txnid_t meta_txnid(const volatile meta_t *meta);
MDBX_INTERNAL txnid_t recent_committed_txnid(const MDBX_env *env);
MDBX_INTERNAL int meta_sync(const MDBX_env *env, const meta_ptr_t head);

MDBX_INTERNAL const char *durable_caption(const meta_t *const meta);
MDBX_INTERNAL void meta_troika_dump(const MDBX_env *env, const troika_t *troika);

#define METAPAGE(env, n) page_meta(pgno2page(env, n))
#define METAPAGE_END(env) METAPAGE(env, NUM_METAS)

static inline meta_ptr_t meta_recent(const MDBX_env *env, const troika_t *troika) {
  meta_ptr_t r;
  r.txnid = troika->txnid[troika->recent];
  r.ptr_v = METAPAGE(env, troika->recent);
  r.is_steady = (troika->fsm >> troika->recent) & 1;
  return r;
}

static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env, const troika_t *troika) {
  meta_ptr_t r;
  r.txnid = troika->txnid[troika->prefer_steady];
  r.ptr_v = METAPAGE(env, troika->prefer_steady);
  r.is_steady = (troika->fsm >> troika->prefer_steady) & 1;
  return r;
}

static inline meta_ptr_t meta_tail(const MDBX_env *env, const troika_t *troika) {
  const uint8_t tail = troika->tail_and_flags & 3;
  MDBX_ANALYSIS_ASSUME(tail < NUM_METAS);
  meta_ptr_t r;
  r.txnid = troika->txnid[tail];
  r.ptr_v = METAPAGE(env, tail);
  r.is_steady = (troika->fsm >> tail) & 1;
  return r;
}

static inline bool meta_is_used(const troika_t *troika, unsigned n) {
  return n == troika->recent || n == troika->prefer_steady;
}

static inline bool meta_bootid_match(const meta_t *meta) {
  return memcmp(&meta->bootid, &globals.bootid, 16) == 0 && (globals.bootid.x | globals.bootid.y) != 0;
}

static inline bool meta_weak_acceptable(const MDBX_env *env, const meta_t *meta, const int lck_exclusive) {
  return lck_exclusive
             ? /* exclusive lock */ meta_bootid_match(meta)
             : /* db already opened */ env->lck_mmap.lck && (env->lck_mmap.lck->envmode.weak & MDBX_RDONLY) == 0;
}

MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t constmeta_txnid(const meta_t *meta) {
  const txnid_t a = unaligned_peek_u64(4, &meta->txnid_a);
  const txnid_t b = unaligned_peek_u64(4, &meta->txnid_b);
  return likely(a == b) ? a : 0;
}

static inline void meta_update_begin(const MDBX_env *env, meta_t *meta, txnid_t txnid) {
  eASSERT0(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
  eASSERT0(env, unaligned_peek_u64(4, meta->txnid_a) < txnid && unaligned_peek_u64(4, meta->txnid_b) < txnid);
  (void)env;
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
  atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, 0, mo_AcquireRelease);
  atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_a, txnid, mo_AcquireRelease);
#else
  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease);
  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease);
  atomic_store32(&meta->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease);
  atomic_store32(&meta->txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease);
#endif
}

static inline void meta_update_end(const MDBX_env *env, meta_t *meta, txnid_t txnid) {
  eASSERT0(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
  eASSERT0(env, unaligned_peek_u64(4, meta->txnid_a) == txnid);
  eASSERT0(env, unaligned_peek_u64(4, meta->txnid_b) < txnid);
  (void)env;
  jitter4testing(true);
  memcpy(&meta->bootid, &globals.bootid, 16);
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
  atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, txnid, mo_AcquireRelease);
#else
  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease);
  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease);
#endif
}

static inline void meta_set_txnid(const MDBX_env *env, meta_t *meta, const txnid_t txnid) {
  eASSERT0(env, !env->dxb_mmap.base || meta < METAPAGE(env, 0) || meta >= METAPAGE_END(env));
  (void)env;
  /* update inconsistently since this function used ONLY for filling meta-image
   * for writing, but not the actual meta-page */
  memcpy(&meta->bootid, &globals.bootid, 16);
  unaligned_poke_u64(4, meta->txnid_a, txnid);
  unaligned_poke_u64(4, meta->txnid_b, txnid);
}

static inline uint8_t meta_cmp2int(txnid_t a, txnid_t b, uint8_t s) {
  return unlikely(a == b) ? 1 * s : (a > b) ? 2 * s : 0 * s;
}

static inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, bool a_steady, bool b_steady) {
  ASSERT(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */);
  return ab_cmp2int > 1 || (ab_cmp2int == 1 && a_steady > b_steady);
}

static inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, bool a_steady, bool b_steady) {
  ASSERT(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */);
  return a_steady > b_steady || (a_steady == b_steady && ab_cmp2int > 1);
}

static inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) {
  return meta_cmp2recent(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady);
}

static inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) {
  return meta_cmp2steady(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady);
}

MDBX_INTERNAL meta_t *meta_init_triplet(const MDBX_env *env, void *buffer);

MDBX_INTERNAL int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number,
                                unsigned *guess_pagesize);

MDBX_INTERNAL int __must_check_result meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest);

MDBX_INTERNAL int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape);

MDBX_INTERNAL int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto);

#if !(defined(_WIN32) || defined(_WIN64))
#define MDBX_WRITETHROUGH_THRESHOLD_DEFAULT 2
#endif

struct iov_ctx {
  MDBX_env *env;
  osal_ioring_t *ior;
  mdbx_filehandle_t fd;
  int err;
#ifndef MDBX_NEED_WRITTEN_RANGE
#define MDBX_NEED_WRITTEN_RANGE 1
#endif /* MDBX_NEED_WRITTEN_RANGE */
#if MDBX_NEED_WRITTEN_RANGE
  pgno_t flush_begin;
  pgno_t flush_end;
#endif /* MDBX_NEED_WRITTEN_RANGE */
  uint64_t coherency_timestamp;
};

MDBX_INTERNAL __must_check_result int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages,
                                               mdbx_filehandle_t fd, bool check_coherence);

static inline bool iov_empty(const iov_ctx_t *ctx) { return osal_ioring_used(ctx->ior) == 0; }

MDBX_INTERNAL __must_check_result int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages);

MDBX_INTERNAL __must_check_result int iov_write(iov_ctx_t *ctx);

MDBX_INTERNAL void spill_remove(MDBX_txn *txn, size_t idx, size_t npages);
MDBX_INTERNAL pnl_t spill_purge(MDBX_txn *txn);
MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries,
                                 const intptr_t wanna_spill_npages, const size_t need);
/*----------------------------------------------------------------------------*/

static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) {
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
  const pnl_t pnl = txn->wr.spilled.list;
  if (likely(!pnl))
    return 0;
  pgno <<= 1;
  size_t n = pnl_search(pnl, pgno, (size_t)MAX_PAGENO + MAX_PAGENO + 1);
  return (n <= pnl_size(pnl) && pnl[n] == pgno) ? n : 0;
}

static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) {
  const pnl_t pnl = txn->wr.spilled.list;
  if (likely(!pnl))
    return false;
  const size_t len = pnl_size(pnl);
  if (LOG_ENABLED(MDBX_LOG_EXTRA)) {
    DEBUG_EXTRA("PNL len %zu [", len);
    for (size_t i = 1; i <= len; ++i)
      DEBUG_EXTRA_PRINT(" %li", (pnl[i] & 1) ? -(long)(pnl[i] >> 1) : (long)(pnl[i] >> 1));
    DEBUG_EXTRA_PRINT("%s\n", "]");
  }
  const pgno_t spilled_range_begin = pgno << 1;
  const pgno_t spilled_range_last = ((pgno + (pgno_t)npages) << 1) - 1;
#if MDBX_PNL_ASCENDING
  const size_t n = pnl_search(pnl, spilled_range_begin, (size_t)(MAX_PAGENO + 1) << 1);
  cASSERT0(txn, n && (n == pnl_size(pnl) + 1 || spilled_range_begin <= pnl[n]));
  const bool rc = n <= pnl_size(pnl) && pnl[n] <= spilled_range_last;
#else
  const size_t n = pnl_search(pnl, spilled_range_last, (size_t)MAX_PAGENO + MAX_PAGENO + 1);
  cASSERT0(txn, n && (n == pnl_size(pnl) + 1 || spilled_range_last >= pnl[n]));
  const bool rc = n <= pnl_size(pnl) && pnl[n] >= spilled_range_begin;
#endif
  if (CHECKS1_ENABLED()) {
    bool check = false;
    for (size_t i = 0; i < npages; ++i)
      check |= spill_search(txn, (pgno_t)(pgno + i)) != 0;
    cASSERT0(txn, check == rc);
  }
  return rc;
}

static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, const size_t need) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  tASSERT1(txn, !m0 || cursor_is_tracked(m0));

  const intptr_t wanna_spill_entries = txn->wr.dirtylist ? (need - txn->wr.dirtyroom - txn->wr.loose_count) : 0;
  const intptr_t wanna_spill_npages =
      need + (txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages) -
      txn->wr.loose_count - txn->env->options.dp_limit;

  /* production mode */
  if (likely(wanna_spill_npages < 1 && wanna_spill_entries < 1)
#if xMDBX_DEBUG_SPILLING == 1
      /* debug mode: always try to spill if xMDBX_DEBUG_SPILLING == 1 */
      && txn->txnid % 23 > 11
#endif
  )
    return MDBX_SUCCESS;

  return spill_slowpath(txn, m0, wanna_spill_entries, wanna_spill_npages, need);
}

#define MDBX_SPLIT_REPLACE MDBX_APPENDDUP /* newkey is not new */
MDBX_INTERNAL int __must_check_result page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata,
                                                 pgno_t newpgno, const unsigned naf);

/*----------------------------------------------------------------------------*/

MDBX_NOTHROW_PURE_FUNCTION static inline bool is_frozen(const MDBX_txn *txn, const page_t *mp) {
  return mp->txnid < txn->txnid;
}

MDBX_NOTHROW_PURE_FUNCTION static inline bool is_spilled(const MDBX_txn *txn, const page_t *mp) {
  return mp->txnid == txn->txnid;
}

MDBX_NOTHROW_PURE_FUNCTION static inline bool is_shadowed(const MDBX_txn *txn, const page_t *mp) {
  return mp->txnid > txn->txnid;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_correct(const MDBX_txn *txn, const page_t *mp) {
  return mp->txnid <= txn->front_txnid;
}

MDBX_NOTHROW_PURE_FUNCTION static inline bool is_modifable(const MDBX_txn *txn, const page_t *mp) {
  return mp->txnid == txn->front_txnid;
}

MDBX_INTERNAL int __must_check_result page_check(const MDBX_cursor *const mc, const page_t *const mp);

MDBX_INTERNAL pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front);

MDBX_INTERNAL pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front);

MDBX_INTERNAL pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front);

static inline int __must_check_result page_get(const MDBX_cursor *mc, const pgno_t pgno, page_t **mp,
                                               const txnid_t front) {
  pgr_t ret = page_get_three(mc, pgno, front);
  *mp = ret.page;
  return ret.err;
}

MDBX_INTERNAL pgr_t page_get_unchecked(MDBX_txn *const txn, const pgno_t pgno, const txnid_t front);

/*----------------------------------------------------------------------------*/

MDBX_INTERNAL int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages);
MDBX_INTERNAL pgr_t page_new(MDBX_cursor *mc, const unsigned flags);
MDBX_INTERNAL pgr_t page_new_large(MDBX_cursor *mc, const size_t npages);
MDBX_INTERNAL int page_touch_modifable(MDBX_txn *txn, const page_t *const mp);
MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp);

static inline int page_touch(MDBX_cursor *mc) {
  page_t *const mp = mc->pg[mc->top];
  MDBX_txn *txn = mc->txn;

  cASSERT0(txn, mc->txn->flags & MDBX_TXN_DIRTY);
  cASSERT0(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
  cASSERT0(txn, !is_largepage(mp));
  if (CHECKS1_ENABLED()) {
    if (mc->flags & z_inner) {
      subcur_t *mx = container_of(mc->tree, subcur_t, nested_tree);
      cursor_couple_t *couple = container_of(mx, cursor_couple_t, inner);
      tASSERT0(txn, mc->tree == &couple->outer.subcur->nested_tree);
      tASSERT0(txn, &mc->clc->k == &couple->outer.clc->v);
      tASSERT0(txn, *couple->outer.dbi_state & DBI_DIRTY);
    }
    tASSERT0(txn, txn_dpl_check(txn));
  }

  if (is_modifable(txn, mp)) {
    if (!txn->wr.dirtylist) {
      cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC);
      return MDBX_SUCCESS;
    }
    return is_subpage(mp) ? MDBX_SUCCESS : page_touch_modifable(txn, mp);
  }
  return page_touch_unmodifable(txn, mc, mp);
}

MDBX_INTERNAL void page_copy(page_t *const dst, const page_t *const src, const size_t size);
MDBX_INTERNAL pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp);

MDBX_INTERNAL page_t *page_shadow_alloc(MDBX_txn *txn, size_t num);

MDBX_INTERNAL void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages);

MDBX_INTERNAL int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */,
                                 unsigned pageflags /* maybe unknown/zero */);

static inline int page_retire(MDBX_cursor *mc, page_t *mp) { return page_retire_ex(mc, mp->pgno, mp, mp->flags); }

static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const size_t npages) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  mp->txnid = INVALID_TXNID;
  mp->flags = P_BAD;

  if (txn->wr.dirtylist) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
    cASSERT0(txn, MDBX_AVOID_MSYNC || (di && txn->wr.dirtylist->items[di].ptr == mp));
    if (!MDBX_AVOID_MSYNC || di) {
      txn_dpl_remove_ex(txn, di, npages);
      txn->wr.dirtyroom++;
      cASSERT0(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
                        (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
      if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) {
        page_shadow_release(txn->env, mp, npages);
        return;
      }
    }
  } else {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && !di);
    txn->wr.writemap_dirty_npages -= (txn->wr.writemap_dirty_npages > npages) ? npages : txn->wr.writemap_dirty_npages;
  }
  VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ);
  VALGRIND_MAKE_MEM_NOACCESS(page2payload(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ);
  MDBX_ASAN_POISON_MEMORY_REGION(page2payload(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ);
}

MDBX_INTERNAL size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len,
                                           size_t item_len);

#define page_next(mp) (*(page_t **)ptr_disp((mp)->entries, sizeof(void *) - sizeof(uint32_t)))

MDBX_INTERNAL void rthc_ctor(void);
MDBX_INTERNAL void rthc_dtor(const mdbx_pid_t current_pid);
MDBX_INTERNAL void rthc_lock(void);
MDBX_INTERNAL void rthc_unlock(void);

MDBX_INTERNAL int rthc_register(MDBX_env *const env);
MDBX_INTERNAL int rthc_remove(MDBX_env *const env);
MDBX_INTERNAL int rthc_uniq_check(const osal_mmap_t *pending, MDBX_env **found);

/* dtor called for thread, i.e. for all mdbx's environment objects */
MDBX_INTERNAL void rthc_thread_dtor(void *rthc);

static inline void *thread_rthc_get(osal_thread_key_t key) {
#if defined(_WIN32) || defined(_WIN64)
  return TlsGetValue(key);
#else
  return pthread_getspecific(key);
#endif
}

MDBX_INTERNAL void thread_rthc_set(osal_thread_key_t key, const void *value);

#if !defined(_WIN32) && !defined(_WIN64)
MDBX_INTERNAL void rthc_afterfork(void);
MDBX_INTERNAL void workaround_glibc_bug21031(void);
#endif /* !Windows */

static inline void thread_key_delete(osal_thread_key_t key) {
  TRACE("key = %" PRIuPTR, (uintptr_t)key);
#if defined(_WIN32) || defined(_WIN64)
  ENSURE(TlsFree(key));
#else
  ENSURE(pthread_key_delete(key) == 0);
  workaround_glibc_bug21031();
#endif
}

typedef struct walk_tbl {
  MDBX_val name;
  tree_t *internal, *nested;
} walk_tbl_t;

typedef int walk_func(const size_t pgno, const unsigned number, void *const ctx, const unsigned deep,
                      const walk_tbl_t *table, const size_t page_size, const page_type_t page_type,
                      const txnid_t page_txnid, const MDBX_error_t err, const size_t nentries,
                      const size_t payload_bytes, const size_t header_bytes, const size_t unused_bytes,
                      const size_t parent_pgno);

typedef enum walk_options { dont_check_keys_ordering = 1, dont_walk_GC = 2, dont_walk_MAIN = 4 } walk_options_t;

MDBX_INTERNAL int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options);

typedef struct walk_ctx {
  void *userctx;
  walk_options_t options;
  unsigned deep;
  walk_func *visitor;
  MDBX_txn *txn;
  MDBX_cursor *cursor;
} walk_ctx_t;

MDBX_INTERNAL int walk_tbl(walk_ctx_t *ctx, walk_tbl_t *tbl, pgno_t parent_page);

#define MDBX_RADIXSORT_THRESHOLD 142

/* ---------------------------------------------------------------------------
 * LY: State of the art quicksort-based sorting, with internal stack
 * and network-sort for small chunks.
 * Thanks to John M. Gamble for the http://pages.ripco.net/~jgamble/nw.html */

#if MDBX_HAVE_CMOV
#define SORT_CMP_SWAP(TYPE, CMP, a, b)                                                                                 \
  do {                                                                                                                 \
    const TYPE swap_tmp = (a);                                                                                         \
    const bool swap_cmp = expect_with_probability(CMP(swap_tmp, b), 0, .5);                                            \
    (a) = swap_cmp ? swap_tmp : b;                                                                                     \
    (b) = swap_cmp ? b : swap_tmp;                                                                                     \
  } while (0)
#else
#define SORT_CMP_SWAP(TYPE, CMP, a, b)                                                                                 \
  do                                                                                                                   \
    if (expect_with_probability(!CMP(a, b), 0, .5)) {                                                                  \
      const TYPE swap_tmp = (a);                                                                                       \
      (a) = (b);                                                                                                       \
      (b) = swap_tmp;                                                                                                  \
    }                                                                                                                  \
  while (0)
#endif

//  3 comparators, 3 parallel operations
//  o-----^--^--o
//        |  |
//  o--^--|--v--o
//     |  |
//  o--v--v-----o
//
//  [[1,2]]
//  [[0,2]]
//  [[0,1]]
#define SORT_NETWORK_3(TYPE, CMP, begin)                                                                               \
  do {                                                                                                                 \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]);                                                                      \
  } while (0)

//  5 comparators, 3 parallel operations
//  o--^--^--------o
//     |  |
//  o--v--|--^--^--o
//        |  |  |
//  o--^--v--|--v--o
//     |     |
//  o--v-----v-----o
//
//  [[0,1],[2,3]]
//  [[0,2],[1,3]]
//  [[1,2]]
#define SORT_NETWORK_4(TYPE, CMP, begin)                                                                               \
  do {                                                                                                                 \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]);                                                                      \
  } while (0)

//  9 comparators, 5 parallel operations
//  o--^--^-----^-----------o
//     |  |     |
//  o--|--|--^--v-----^--^--o
//     |  |  |        |  |
//  o--|--v--|--^--^--|--v--o
//     |     |  |  |  |
//  o--|-----v--|--v--|--^--o
//     |        |     |  |
//  o--v--------v-----v--v--o
//
//  [[0,4],[1,3]]
//  [[0,2]]
//  [[2,4],[0,1]]
//  [[2,3],[1,4]]
//  [[1,2],[3,4]]
#define SORT_NETWORK_5(TYPE, CMP, begin)                                                                               \
  do {                                                                                                                 \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]);                                                                      \
  } while (0)

//  12 comparators, 6 parallel operations
//  o-----^--^--^-----------------o
//        |  |  |
//  o--^--|--v--|--^--------^-----o
//     |  |     |  |        |
//  o--v--v-----|--|--^--^--|--^--o
//              |  |  |  |  |  |
//  o-----^--^--v--|--|--|--v--v--o
//        |  |     |  |  |
//  o--^--|--v-----v--|--v--------o
//     |  |           |
//  o--v--v-----------v-----------o
//
//  [[1,2],[4,5]]
//  [[0,2],[3,5]]
//  [[0,1],[3,4],[2,5]]
//  [[0,3],[1,4]]
//  [[2,4],[1,3]]
//  [[2,3]]
#define SORT_NETWORK_6(TYPE, CMP, begin)                                                                               \
  do {                                                                                                                 \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]);                                                                      \
  } while (0)

//  16 comparators, 6 parallel operations
//  o--^--------^-----^-----------------o
//     |        |     |
//  o--|--^-----|--^--v--------^--^-----o
//     |  |     |  |           |  |
//  o--|--|--^--v--|--^-----^--|--v-----o
//     |  |  |     |  |     |  |
//  o--|--|--|-----v--|--^--v--|--^--^--o
//     |  |  |        |  |     |  |  |
//  o--v--|--|--^-----v--|--^--v--|--v--o
//        |  |  |        |  |     |
//  o-----v--|--|--------v--v-----|--^--o
//           |  |                 |  |
//  o--------v--v-----------------v--v--o
//
//  [[0,4],[1,5],[2,6]]
//  [[0,2],[1,3],[4,6]]
//  [[2,4],[3,5],[0,1]]
//  [[2,3],[4,5]]
//  [[1,4],[3,6]]
//  [[1,2],[3,4],[5,6]]
#define SORT_NETWORK_7(TYPE, CMP, begin)                                                                               \
  do {                                                                                                                 \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]);                                                                      \
  } while (0)

//  19 comparators, 6 parallel operations
//  o--^--------^-----^-----------------o
//     |        |     |
//  o--|--^-----|--^--v--------^--^-----o
//     |  |     |  |           |  |
//  o--|--|--^--v--|--^-----^--|--v-----o
//     |  |  |     |  |     |  |
//  o--|--|--|--^--v--|--^--v--|--^--^--o
//     |  |  |  |     |  |     |  |  |
//  o--v--|--|--|--^--v--|--^--v--|--v--o
//        |  |  |  |     |  |     |
//  o-----v--|--|--|--^--v--v-----|--^--o
//           |  |  |  |           |  |
//  o--------v--|--v--|--^--------v--v--o
//              |     |  |
//  o-----------v-----v--v--------------o
//
//  [[0,4],[1,5],[2,6],[3,7]]
//  [[0,2],[1,3],[4,6],[5,7]]
//  [[2,4],[3,5],[0,1],[6,7]]
//  [[2,3],[4,5]]
//  [[1,4],[3,6]]
//  [[1,2],[3,4],[5,6]]
#define SORT_NETWORK_8(TYPE, CMP, begin)                                                                               \
  do {                                                                                                                 \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[7]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[7]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[6], begin[7]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]);                                                                      \
    SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]);                                                                      \
  } while (0)

#define SORT_INNER(TYPE, CMP, begin, end, len)                                                                         \
  switch (len) {                                                                                                       \
  default:                                                                                                             \
    ASSERT(false);                                                                                                     \
    __unreachable();                                                                                                   \
  case 0:                                                                                                              \
  case 1:                                                                                                              \
    break;                                                                                                             \
  case 2:                                                                                                              \
    SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]);                                                                      \
    break;                                                                                                             \
  case 3:                                                                                                              \
    SORT_NETWORK_3(TYPE, CMP, begin);                                                                                  \
    break;                                                                                                             \
  case 4:                                                                                                              \
    SORT_NETWORK_4(TYPE, CMP, begin);                                                                                  \
    break;                                                                                                             \
  case 5:                                                                                                              \
    SORT_NETWORK_5(TYPE, CMP, begin);                                                                                  \
    break;                                                                                                             \
  case 6:                                                                                                              \
    SORT_NETWORK_6(TYPE, CMP, begin);                                                                                  \
    break;                                                                                                             \
  case 7:                                                                                                              \
    SORT_NETWORK_7(TYPE, CMP, begin);                                                                                  \
    break;                                                                                                             \
  case 8:                                                                                                              \
    SORT_NETWORK_8(TYPE, CMP, begin);                                                                                  \
    break;                                                                                                             \
  }

#define SORT_SWAP(TYPE, a, b)                                                                                          \
  do {                                                                                                                 \
    const TYPE swap_tmp = (a);                                                                                         \
    (a) = (b);                                                                                                         \
    (b) = swap_tmp;                                                                                                    \
  } while (0)

#define SORT_PUSH(low, high)                                                                                           \
  do {                                                                                                                 \
    top->lo = (low);                                                                                                   \
    top->hi = (high);                                                                                                  \
    ++top;                                                                                                             \
  } while (0)

#define SORT_POP(low, high)                                                                                            \
  do {                                                                                                                 \
    --top;                                                                                                             \
    low = top->lo;                                                                                                     \
    high = top->hi;                                                                                                    \
  } while (0)

#define SORT_IMPL(NAME, EXPECT_LOW_CARDINALITY_OR_PRESORTED, TYPE, CMP)                                                \
                                                                                                                       \
  static inline bool NAME##_is_sorted(const TYPE *first, const TYPE *last) {                                           \
    while (++first <= last)                                                                                            \
      if (expect_with_probability(CMP(first[0], first[-1]), 1, .1))                                                    \
        return false;                                                                                                  \
    return true;                                                                                                       \
  }                                                                                                                    \
                                                                                                                       \
  typedef struct {                                                                                                     \
    TYPE *lo, *hi;                                                                                                     \
  } NAME##_stack;                                                                                                      \
                                                                                                                       \
  __hot static void NAME(TYPE *const __restrict begin, TYPE *const __restrict end) {                                   \
    NAME##_stack stack[sizeof(size_t) * CHAR_BIT], *__restrict top = stack;                                            \
                                                                                                                       \
    TYPE *__restrict hi = end - 1;                                                                                     \
    TYPE *__restrict lo = begin;                                                                                       \
    while (true) {                                                                                                     \
      const ptrdiff_t len = hi - lo;                                                                                   \
      if (len < 8) {                                                                                                   \
        SORT_INNER(TYPE, CMP, lo, hi + 1, len + 1);                                                                    \
        if (unlikely(top == stack))                                                                                    \
          break;                                                                                                       \
        SORT_POP(lo, hi);                                                                                              \
        continue;                                                                                                      \
      }                                                                                                                \
                                                                                                                       \
      TYPE *__restrict mid = lo + (len >> 1);                                                                          \
      SORT_CMP_SWAP(TYPE, CMP, *lo, *mid);                                                                             \
      SORT_CMP_SWAP(TYPE, CMP, *mid, *hi);                                                                             \
      SORT_CMP_SWAP(TYPE, CMP, *lo, *mid);                                                                             \
                                                                                                                       \
      TYPE *right = hi - 1;                                                                                            \
      TYPE *left = lo + 1;                                                                                             \
      while (1) {                                                                                                      \
        while (expect_with_probability(CMP(*left, *mid), 0, .5))                                                       \
          ++left;                                                                                                      \
        while (expect_with_probability(CMP(*mid, *right), 0, .5))                                                      \
          --right;                                                                                                     \
        if (unlikely(left > right)) {                                                                                  \
          if (EXPECT_LOW_CARDINALITY_OR_PRESORTED) {                                                                   \
            if (NAME##_is_sorted(lo, right))                                                                           \
              lo = right + 1;                                                                                          \
            if (NAME##_is_sorted(left, hi))                                                                            \
              hi = left;                                                                                               \
          }                                                                                                            \
          break;                                                                                                       \
        }                                                                                                              \
        SORT_SWAP(TYPE, *left, *right);                                                                                \
        mid = (mid == left) ? right : (mid == right) ? left : mid;                                                     \
        ++left;                                                                                                        \
        --right;                                                                                                       \
      }                                                                                                                \
                                                                                                                       \
      if (right - lo > hi - left) {                                                                                    \
        SORT_PUSH(lo, right);                                                                                          \
        lo = left;                                                                                                     \
      } else {                                                                                                         \
        SORT_PUSH(left, hi);                                                                                           \
        hi = right;                                                                                                    \
      }                                                                                                                \
    }                                                                                                                  \
                                                                                                                       \
    if (CHECKS2_ENABLED()) {                                                                                           \
      for (TYPE *scan = begin + 1; scan < end; ++scan)                                                                 \
        ASSERT(CMP(scan[-1], scan[0]));                                                                                \
    }                                                                                                                  \
  }

/*------------------------------------------------------------------------------
 * LY: radix sort for large chunks */

#define RADIXSORT_IMPL(NAME, TYPE, EXTRACT_KEY, BUFFER_PREALLOCATED, END_GAP)                                          \
                                                                                                                       \
  __hot static bool NAME##_radixsort(TYPE *const begin, const size_t length) {                                         \
    TYPE *tmp;                                                                                                         \
    if (BUFFER_PREALLOCATED) {                                                                                         \
      tmp = begin + length + END_GAP;                                                                                  \
      /* memset(tmp, 0xDeadBeef, sizeof(TYPE) * length); */                                                            \
    } else {                                                                                                           \
      tmp = osal_malloc(sizeof(TYPE) * length);                                                                        \
      if (unlikely(!tmp))                                                                                              \
        return false;                                                                                                  \
    }                                                                                                                  \
                                                                                                                       \
    size_t key_shift = 0, key_diff_mask;                                                                               \
    do {                                                                                                               \
      struct {                                                                                                         \
        pgno_t a[256], b[256];                                                                                         \
      } counters;                                                                                                      \
      memset(&counters, 0, sizeof(counters));                                                                          \
                                                                                                                       \
      key_diff_mask = 0;                                                                                               \
      size_t prev_key = EXTRACT_KEY(begin) >> key_shift;                                                               \
      TYPE *r = begin, *end = begin + length;                                                                          \
      do {                                                                                                             \
        const size_t key = EXTRACT_KEY(r) >> key_shift;                                                                \
        counters.a[key & 255]++;                                                                                       \
        counters.b[(key >> 8) & 255]++;                                                                                \
        key_diff_mask |= prev_key ^ key;                                                                               \
        prev_key = key;                                                                                                \
      } while (++r != end);                                                                                            \
                                                                                                                       \
      pgno_t ta = 0, tb = 0;                                                                                           \
      for (size_t i = 0; i < 256; ++i) {                                                                               \
        const pgno_t ia = counters.a[i];                                                                               \
        counters.a[i] = ta;                                                                                            \
        ta += ia;                                                                                                      \
        const pgno_t ib = counters.b[i];                                                                               \
        counters.b[i] = tb;                                                                                            \
        tb += ib;                                                                                                      \
      }                                                                                                                \
                                                                                                                       \
      r = begin;                                                                                                       \
      do {                                                                                                             \
        const size_t key = EXTRACT_KEY(r) >> key_shift;                                                                \
        tmp[counters.a[key & 255]++] = *r;                                                                             \
      } while (++r != end);                                                                                            \
                                                                                                                       \
      if (unlikely(key_diff_mask < 256)) {                                                                             \
        memcpy(begin, tmp, ptr_dist(end, begin));                                                                      \
        break;                                                                                                         \
      }                                                                                                                \
      end = (r = tmp) + length;                                                                                        \
      do {                                                                                                             \
        const size_t key = EXTRACT_KEY(r) >> key_shift;                                                                \
        begin[counters.b[(key >> 8) & 255]++] = *r;                                                                    \
      } while (++r != end);                                                                                            \
                                                                                                                       \
      key_shift += 16;                                                                                                 \
    } while (key_diff_mask >> 16);                                                                                     \
                                                                                                                       \
    if (!(BUFFER_PREALLOCATED))                                                                                        \
      osal_free(tmp);                                                                                                  \
    return true;                                                                                                       \
  }

/*------------------------------------------------------------------------------
 * LY: Binary search */

#if defined(__clang__) && __clang_major__ > 4 && defined(__ia32__)
#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag)                                                                 \
  do                                                                                                                   \
    __asm __volatile(""                                                                                                \
                     : "+r"(size)                                                                                      \
                     : "r" /* the `b` constraint is more suitable here, but                                            \
                              cause CLANG to allocate and push/pop an one more                                         \
                              register, so using the `r` which avoids this. */                                         \
                     (flag));                                                                                          \
  while (0)
#else
#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag)                                                                 \
  do {                                                                                                                 \
    /* nope for non-clang or non-x86 */;                                                                               \
  } while (0)
#endif /* Workaround for CLANG */

#define SEARCH_IMPL(NAME, TYPE_LIST, TYPE_ARG, CMP)                            \
  static __always_inline const TYPE_LIST *NAME(                                \
      const TYPE_LIST *it, size_t length, const TYPE_ARG item) {               \
    const TYPE_LIST *const begin = it, *const end = begin + length;            \
                                                                               \
    if (MDBX_HAVE_CMOV)                                                        \
      do {                                                                     \
        /* Адаптивно-упрощенный шаг двоичного поиска:                          \
         *  - без переходов при наличии cmov или аналога;                      \
         *  - допускает лишние итерации;                                       \
         *  - но ищет пока size > 2, что требует дозавершения поиска           \
         *    среди остающихся 0-1-2 элементов. */                             \
        const TYPE_LIST *const middle = it + (length >> 1);                    \
        length = (length + 1) >> 1;                                            \
        const bool flag = expect_with_probability(CMP(*middle, item), 0, .5);  \
        WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(length, flag);                      \
        it = flag ? middle : it;                                               \
      } while (length > 2);                                                    \
    else                                                                       \
      while (length > 2) {                                                     \
        /* Вариант с использованием условного перехода. Основное отличие в     \
         * том, что при "не равно" (true от компаратора) переход делается на 1 \
         * ближе к концу массива. Алгоритмически это верно и обеспечивает      \
         * чуть-чуть более быструю сходимость, но зато требует больше          \
         * вычислений при true от компаратора. Также ВАЖНО(!) не допускается   \
         * спекулятивное выполнение при size == 0. */                          \
        const TYPE_LIST *const middle = it + (length >> 1);                    \
        length = (length + 1) >> 1;                                            \
        const bool flag = expect_with_probability(CMP(*middle, item), 0, .5);  \
        if (flag) {                                                            \
          it = middle + 1;                                                     \
          length -= 1;                                                         \
        }                                                                      \
      }                                                                        \
    it += length > 1 && expect_with_probability(CMP(*it, item), 0, .5);        \
    it += length > 0 && expect_with_probability(CMP(*it, item), 0, .5);        \
                                                                               \
    if (CHECKS2_ENABLED()) {                                                   \
      for (const TYPE_LIST *scan = begin; scan < it; ++scan)                   \
        ASSERT(CMP(*scan, item));                                              \
      for (const TYPE_LIST *scan = it; scan < end; ++scan)                     \
        ASSERT(!CMP(*scan, item));                                             \
      (void)begin, (void)end;                                                  \
    }                                                                          \
                                                                               \
    return it;                                                                 \
  }

#endif /* !__cplusplus */

#ifdef _MSC_VER
#pragma warning(pop)
#endif /* MSVC */

__cold size_t mdbx_default_pagesize(void) {
  size_t pagesize = globals.sys_pagesize;
  ENSURE(is_powerof2(pagesize));
  pagesize = (pagesize >= MDBX_MIN_PAGESIZE) ? pagesize : MDBX_MIN_PAGESIZE;
  pagesize = (pagesize <= MDBX_MAX_PAGESIZE) ? pagesize : MDBX_MAX_PAGESIZE;
  return pagesize;
}

__cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) {
  if (pagesize < 1)
    pagesize = (intptr_t)mdbx_default_pagesize();
  else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
                    !is_powerof2((size_t)pagesize)))
    return -1;

  return MIN_PAGENO * pagesize;
}

__cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) {
  if (pagesize < 1)
    pagesize = (intptr_t)mdbx_default_pagesize();
  else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
                    !is_powerof2((size_t)pagesize)))
    return -1;

  STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX);
  const uint64_t limit = (1 + (uint64_t)MAX_PAGENO) * pagesize;
  return (limit < MAX_MAPSIZE) ? (intptr_t)limit : (intptr_t)MAX_MAPSIZE;
}

__cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) {
  if (pagesize < 1)
    pagesize = (intptr_t)mdbx_default_pagesize();
  else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
                    !is_powerof2((size_t)pagesize)))
    return -1;

  STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX);
  const uint64_t pgl_limit = pagesize * (uint64_t)(PAGELIST_LIMIT / MDBX_GOLD_RATIO_DBL);
  const uint64_t map_limit = (uint64_t)(MAX_MAPSIZE / MDBX_GOLD_RATIO_DBL);
  return (pgl_limit < map_limit) ? (intptr_t)pgl_limit : (intptr_t)map_limit;
}

__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags) {
  if (pagesize < 1)
    pagesize = (intptr_t)mdbx_default_pagesize();
  if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
               !is_powerof2((size_t)pagesize)))
    return -1;

  return keysize_max(pagesize, flags);
}

__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags) {
  if (unlikely(!env || env->signature.weak != env_signature))
    return -1;

  return (int)mdbx_limits_keysize_max((intptr_t)env->ps, flags);
}

__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) { return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT); }

__cold intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags) { return keysize_min(flags); }

__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags) {
  if (pagesize < 1)
    pagesize = (intptr_t)mdbx_default_pagesize();
  if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
               !is_powerof2((size_t)pagesize)))
    return -1;

  return valsize_max(pagesize, flags);
}

__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags) {
  if (unlikely(!env || env->signature.weak != env_signature))
    return -1;

  return (int)mdbx_limits_valsize_max((intptr_t)env->ps, flags);
}

__cold intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags) { return valsize_min(flags); }

__cold intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) {
  if (pagesize < 1)
    pagesize = (intptr_t)mdbx_default_pagesize();
  if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
               !is_powerof2((size_t)pagesize)))
    return -1;

  if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP))
    return BRANCH_NODE_MAX(pagesize) - NODESIZE;

  return LEAF_NODE_MAX(pagesize) - NODESIZE;
}

__cold int mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) {
  if (unlikely(!env || env->signature.weak != env_signature))
    return -1;

  return (int)mdbx_limits_pairsize4page_max((intptr_t)env->ps, flags);
}

__cold intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) {
  if (pagesize < 1)
    pagesize = (intptr_t)mdbx_default_pagesize();
  if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
               !is_powerof2((size_t)pagesize)))
    return -1;

  if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP))
    return valsize_max(pagesize, flags);

  return PAGESPACE(pagesize);
}

__cold int mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) {
  if (unlikely(!env || env->signature.weak != env_signature))
    return -1;

  return (int)mdbx_limits_valsize4page_max((intptr_t)env->ps, flags);
}

/*----------------------------------------------------------------------------*/

static size_t estimate_rss(size_t database_bytes) {
  return database_bytes + database_bytes / 64 + (512 + MDBX_WORDBITS * 16) * MEGABYTE;
}

__cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, MDBX_warmup_flags_t flags,
                           unsigned timeout_seconds_16dot16) {
  if (unlikely(env == nullptr && txn == nullptr))
    return LOG_IFERR(MDBX_EINVAL);
  if (unlikely(flags > (MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock | MDBX_warmup_touchlimit |
                        MDBX_warmup_release)))
    return LOG_IFERR(MDBX_EINVAL);

  if (txn) {
    int err = check_txn(txn, MDBX_TXN_FINISHED | MDBX_TXN_ERROR);
    if (unlikely(err != MDBX_SUCCESS))
      return LOG_IFERR(err);
  }
  if (env) {
    int err = check_env(env, false);
    if (unlikely(err != MDBX_SUCCESS))
      return LOG_IFERR(err);
    if (txn && unlikely(txn->env != env))
      return LOG_IFERR(MDBX_EINVAL);
  } else {
    env = txn->env;
  }

  const uint64_t timeout_monotime = (timeout_seconds_16dot16 && (flags & MDBX_warmup_force))
                                        ? osal_monotime() + osal_16dot16_to_monotime(timeout_seconds_16dot16)
                                        : 0;

  if (flags & MDBX_warmup_release)
    munlock_all(env);

  pgno_t used_pgno;
  if (txn) {
    used_pgno = txn->geo.first_unallocated;
  } else {
    const troika_t troika = meta_tap(env);
    used_pgno = meta_recent(env, &troika).ptr_v->geometry.first_unallocated;
  }
  const size_t used_range = pgno_ceil2os_bytes(env, used_pgno);
  const pgno_t mlock_pgno = bytes2pgno(env, used_range);

  int rc = MDBX_SUCCESS;
  if (flags & MDBX_warmup_touchlimit) {
    const size_t estimated_rss = estimate_rss(used_range);
#if defined(_WIN32) || defined(_WIN64)
    SIZE_T current_ws_lower, current_ws_upper;
    if (GetProcessWorkingSetSize(GetCurrentProcess(), &current_ws_lower, &current_ws_upper) &&
        current_ws_lower < estimated_rss) {
      const SIZE_T ws_lower = estimated_rss;
      const SIZE_T ws_upper =
          (MDBX_WORDBITS == 32 && ws_lower > MEGABYTE * 2048) ? ws_lower : ws_lower + MDBX_WORDBITS * MEGABYTE * 32;
      if (!SetProcessWorkingSetSize(GetCurrentProcess(), ws_lower, ws_upper)) {
        rc = (int)GetLastError();
        WARNING("SetProcessWorkingSetSize(%zu, %zu) error %d", ws_lower, ws_upper, rc);
      }
    }
#endif /* Windows */
#ifdef RLIMIT_RSS
    struct rlimit rss;
    if (getrlimit(RLIMIT_RSS, &rss) == 0 && rss.rlim_cur < estimated_rss) {
      rss.rlim_cur = estimated_rss;
      if (rss.rlim_max < estimated_rss)
        rss.rlim_max = estimated_rss;
      if (setrlimit(RLIMIT_RSS, &rss)) {
        rc = errno;
        WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_RSS", (size_t)rss.rlim_cur, (size_t)rss.rlim_max, rc);
      }
    }
#endif /* RLIMIT_RSS */
#ifdef RLIMIT_MEMLOCK
    if (flags & MDBX_warmup_lock) {
      struct rlimit memlock;
      if (getrlimit(RLIMIT_MEMLOCK, &memlock) == 0 && memlock.rlim_cur < estimated_rss) {
        memlock.rlim_cur = estimated_rss;
        if (memlock.rlim_max < estimated_rss)
          memlock.rlim_max = estimated_rss;
        if (setrlimit(RLIMIT_MEMLOCK, &memlock)) {
          rc = errno;
          WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_MEMLOCK", (size_t)memlock.rlim_cur,
                  (size_t)memlock.rlim_max, rc);
        }
      }
    }
#endif /* RLIMIT_MEMLOCK */
    (void)estimated_rss;
  }

#if defined(MLOCK_ONFAULT) &&                                                                                          \
    ((defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 27)) || (defined(__ANDROID_API__) && __ANDROID_API__ >= 30)) &&        \
    (defined(__linux__) || defined(__gnu_linux__))
  if ((flags & MDBX_warmup_lock) != 0 && globals.linux_kernel_version >= 0x04040000 &&
      atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) {
    if (mlock2(env->dxb_mmap.base, used_range, MLOCK_ONFAULT)) {
      rc = errno;
      WARNING("mlock2(%zu, %s) error %d", used_range, "MLOCK_ONFAULT", rc);
    } else {
      update_mlcnt(env, mlock_pgno, true);
      rc = MDBX_SUCCESS;
    }
    if (rc != EINVAL)
      flags -= MDBX_warmup_lock;
  }
#endif /* MLOCK_ONFAULT */

  int err = MDBX_ENOSYS;
  err = dxb_set_readahead(env, used_pgno, true, true);
  if (err != MDBX_SUCCESS && rc == MDBX_SUCCESS)
    rc = err;

  if ((flags & MDBX_warmup_force) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS)) {
    const volatile uint8_t *ptr = env->dxb_mmap.base;
    size_t offset = 0, unused = 42;
#if !(defined(_WIN32) || defined(_WIN64))
    if (flags & MDBX_warmup_oomsafe) {
      const int null_fd = open("/dev/null", O_WRONLY);
      if (unlikely(null_fd < 0))
        rc = errno;
      else {
        struct iovec iov[MDBX_AUXILARY_IOV_MAX];
        for (;;) {
          unsigned i;
          for (i = 0; i < MDBX_AUXILARY_IOV_MAX && offset < used_range; ++i) {
            iov[i].iov_base = (void *)(ptr + offset);
            iov[i].iov_len = 1;
            offset += globals.sys_pagesize;
          }
          if (unlikely(writev(null_fd, iov, i) < 0)) {
            rc = errno;
            if (rc == EFAULT)
              rc = ENOMEM;
            break;
          }
          if (offset >= used_range) {
            rc = MDBX_SUCCESS;
            break;
          }
          if (timeout_seconds_16dot16 && osal_monotime() > timeout_monotime) {
            rc = MDBX_RESULT_TRUE;
            break;
          }
        }
        close(null_fd);
      }
    } else
#endif /* Windows */
      for (;;) {
        unused += ptr[offset];
        offset += globals.sys_pagesize;
        if (offset >= used_range) {
          rc = MDBX_SUCCESS;
          break;
        }
        if (timeout_seconds_16dot16 && osal_monotime() > timeout_monotime) {
          rc = MDBX_RESULT_TRUE;
          break;
        }
      }
    (void)unused;
  }

  if ((flags & MDBX_warmup_lock) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS) &&
      atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) {
#if defined(_WIN32) || defined(_WIN64)
    if (VirtualLock(env->dxb_mmap.base, used_range)) {
      update_mlcnt(env, mlock_pgno, true);
      rc = MDBX_SUCCESS;
    } else {
      rc = (int)GetLastError();
      WARNING("%s(%zu) error %d", "VirtualLock", used_range, rc);
    }
#elif defined(_POSIX_MEMLOCK_RANGE)
    if (mlock(env->dxb_mmap.base, used_range) == 0) {
      update_mlcnt(env, mlock_pgno, true);
      rc = MDBX_SUCCESS;
    } else {
      rc = errno;
      WARNING("%s(%zu) error %d", "mlock", used_range, rc);
    }
#else
    rc = MDBX_ENOSYS;
#endif
  }

  return LOG_IFERR(rc);
}

/*----------------------------------------------------------------------------*/

__cold int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!arg))
    return LOG_IFERR(MDBX_EINVAL);

  *arg = env->lazy_fd;
  return MDBX_SUCCESS;
}

__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, bool onoff) {
  int rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(flags & ((env->flags & ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS : ~ENV_USABLE_FLAGS)))
    return LOG_IFERR(MDBX_EPERM);

  if (unlikely(env->flags & MDBX_RDONLY))
    return LOG_IFERR(MDBX_EACCESS);

  const bool lock_needed = (env->flags & ENV_ACTIVE) && !env_owned_wrtxn(env);
  bool should_unlock = false;
  if (lock_needed) {
    rc = lck_txn_lock(env, false);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    should_unlock = true;
  }

  if (onoff)
    env->flags = combine_durability_flags(env->flags, flags);
  else
    env->flags &= ~flags;

  if (should_unlock)
    lck_txn_unlock(env);
  return MDBX_SUCCESS;
}

__cold int mdbx_env_get_flags(const MDBX_env *env, unsigned *flags) {
  int rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!flags))
    return LOG_IFERR(MDBX_EINVAL);

  *flags = env->flags & ENV_USABLE_FLAGS;
  return MDBX_SUCCESS;
}

__cold int mdbx_env_set_userctx(MDBX_env *env, void *ctx) {
  int rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  env->userctx = ctx;
  return MDBX_SUCCESS;
}

__cold void *mdbx_env_get_userctx(const MDBX_env *env) { return env ? env->userctx : nullptr; }

__cold void mdbx_set_panic(MDBX_panic_func func) { globals.panic_func = func; }

__cold int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func hsr) {
  int rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  env->hsr_callback = hsr;
  return MDBX_SUCCESS;
}

__cold MDBX_hsr_func mdbx_env_get_hsr(const MDBX_env *env) {
  return likely(env && env->signature.weak == env_signature) ? env->hsr_callback : nullptr;
}

#if defined(_WIN32) || defined(_WIN64)
__cold int mdbx_env_get_pathW(const MDBX_env *env, const wchar_t **arg) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!arg))
    return LOG_IFERR(MDBX_EINVAL);

  *arg = env->pathname.specified;
  return MDBX_SUCCESS;
}
#endif /* Windows */

__cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!arg))
    return LOG_IFERR(MDBX_EINVAL);

#if defined(_WIN32) || defined(_WIN64)
  if (!env->pathname_char) {
    *arg = nullptr;
    DWORD flags = /* WC_ERR_INVALID_CHARS */ 0x80;
    size_t mb_len =
        WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr);
    rc = mb_len ? MDBX_SUCCESS : (int)GetLastError();
    if (rc == ERROR_INVALID_FLAGS) {
      mb_len = WideCharToMultiByte(CP_THREAD_ACP, flags = 0, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr);
      rc = mb_len ? MDBX_SUCCESS : (int)GetLastError();
    }
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);

    char *const mb_pathname = osal_malloc(mb_len);
    if (!mb_pathname)
      return LOG_IFERR(MDBX_ENOMEM);
    if (mb_len != (size_t)WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, mb_pathname,
                                              (int)mb_len, nullptr, nullptr)) {
      rc = (int)GetLastError();
      osal_free(mb_pathname);
      return LOG_IFERR(rc);
    }
    if (env->pathname_char ||
        InterlockedCompareExchangePointer((PVOID volatile *)&env->pathname_char, mb_pathname, nullptr))
      osal_free(mb_pathname);
  }
  *arg = env->pathname_char;
#else
  *arg = env->pathname.specified;
#endif /* Windows */
  return MDBX_SUCCESS;
}

/*------------------------------------------------------------------------------
 * Legacy API */

#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API

LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret) {
  return __inline_mdbx_txn_begin(env, parent, flags, ret);
}

LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn) { return __inline_mdbx_txn_commit(txn); }

LIBMDBX_API int mdbx_txn_abort(MDBX_txn *txn) { return __inline_mdbx_txn_abort(txn); }

LIBMDBX_API __cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, size_t bytes) {
  return __inline_mdbx_env_stat(env, stat, bytes);
}

LIBMDBX_API __cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info, size_t bytes) {
  return __inline_mdbx_env_info(env, info, bytes);
}

LIBMDBX_API int mdbx_dbi_flags(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags) {
  return __inline_mdbx_dbi_flags(txn, dbi, flags);
}

LIBMDBX_API __cold int mdbx_env_sync(MDBX_env *env) { return __inline_mdbx_env_sync(env); }

LIBMDBX_API __cold int mdbx_env_sync_poll(MDBX_env *env) { return __inline_mdbx_env_sync_poll(env); }

LIBMDBX_API __cold int mdbx_env_close(MDBX_env *env) { return __inline_mdbx_env_close(env); }

LIBMDBX_API __cold int mdbx_env_set_mapsize(MDBX_env *env, size_t size) {
  return __inline_mdbx_env_set_mapsize(env, size);
}

LIBMDBX_API __cold int mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) {
  return __inline_mdbx_env_set_maxdbs(env, dbs);
}

LIBMDBX_API __cold int mdbx_env_get_maxdbs(const MDBX_env *env, MDBX_dbi *dbs) {
  return __inline_mdbx_env_get_maxdbs(env, dbs);
}

LIBMDBX_API __cold int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) {
  return __inline_mdbx_env_set_maxreaders(env, readers);
}

LIBMDBX_API __cold int mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers) {
  return __inline_mdbx_env_get_maxreaders(env, readers);
}

LIBMDBX_API __cold int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) {
  return __inline_mdbx_env_set_syncbytes(env, threshold);
}

LIBMDBX_API __cold int mdbx_env_get_syncbytes(const MDBX_env *env, size_t *threshold) {
  return __inline_mdbx_env_get_syncbytes(env, threshold);
}

LIBMDBX_API __cold int mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) {
  return __inline_mdbx_env_set_syncperiod(env, seconds_16dot16);
}

LIBMDBX_API __cold int mdbx_env_get_syncperiod(const MDBX_env *env, unsigned *seconds_16dot16) {
  return __inline_mdbx_env_get_syncperiod(env, seconds_16dot16);
}

LIBMDBX_API __cold uint64_t mdbx_key_from_int64(const int64_t i64) { return __inline_mdbx_key_from_int64(i64); }

LIBMDBX_API __cold uint32_t mdbx_key_from_int32(const int32_t i32) { return __inline_mdbx_key_from_int32(i32); }

LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_min(void) { return __inline_mdbx_limits_pgsize_min(); }

LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_max(void) { return __inline_mdbx_limits_pgsize_max(); }

#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */

typedef struct compacting_context {
  MDBX_env *env;
  MDBX_txn *txn;
  MDBX_copy_flags_t flags;
  pgno_t first_unallocated;
  osal_condpair_t condpair;
  volatile unsigned head;
  volatile unsigned tail;
  uint8_t *write_buf[2];
  size_t write_len[2];
  /* Error code.  Never cleared if set.  Both threads can set nonzero
   * to fail the copy.  Not mutex-protected, expects atomic int. */
  volatile int error;
  mdbx_filehandle_t fd;
} ctx_t;

__cold static int compacting_walk_tree(ctx_t *ctx, tree_t *tree);

/* Dedicated writer thread for compacting copy. */
__cold static THREAD_RESULT THREAD_CALL compacting_write_thread(void *arg) {
  ctx_t *const ctx = arg;

#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64))
  sigset_t sigset;
  sigemptyset(&sigset);
  sigaddset(&sigset, SIGPIPE);
  ctx->error = pthread_sigmask(SIG_BLOCK, &sigset, nullptr);
#endif /* EPIPE */

  osal_condpair_lock(&ctx->condpair);
  while (!ctx->error) {
    while (ctx->tail == ctx->head && !ctx->error) {
      int err = osal_condpair_wait(&ctx->condpair, true);
      if (err != MDBX_SUCCESS) {
        ctx->error = err;
        goto bailout;
      }
    }
    const unsigned toggle = ctx->tail & 1;
    size_t wsize = ctx->write_len[toggle];
    if (wsize == 0) {
      ctx->tail += 1;
      break /* EOF */;
    }
    ctx->write_len[toggle] = 0;
    uint8_t *ptr = ctx->write_buf[toggle];
    if (!ctx->error) {
      int err = osal_write(ctx->fd, ptr, wsize);
      if (err != MDBX_SUCCESS) {
#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64))
        if (err == EPIPE) {
          /* Collect the pending SIGPIPE,
           * otherwise at least OS X gives it to the process on thread-exit. */
          int unused;
          sigwait(&sigset, &unused);
        }
#endif /* EPIPE */
        ctx->error = err;
        goto bailout;
      }
    }
    ctx->tail += 1;
    osal_condpair_signal(&ctx->condpair, false);
  }
bailout:
  osal_condpair_unlock(&ctx->condpair);
  return (THREAD_RESULT)0;
}

/* Give buffer and/or MDBX_EOF to writer thread, await unused buffer. */
__cold static int compacting_toggle_write_buffers(ctx_t *ctx) {
  osal_condpair_lock(&ctx->condpair);
  eASSERT0(ctx->env, ctx->head - ctx->tail < 2 || ctx->error);
  ctx->head += 1;
  osal_condpair_signal(&ctx->condpair, true);
  while (!ctx->error && ctx->head - ctx->tail == 2 /* both buffers in use */) {
    if (ctx->flags & MDBX_CP_THROTTLE_MVCC)
      mdbx_txn_park(ctx->txn, false);
    int err = osal_condpair_wait(&ctx->condpair, false);
    if (err == MDBX_SUCCESS && (ctx->flags & MDBX_CP_THROTTLE_MVCC) != 0)
      err = mdbx_txn_unpark(ctx->txn, false);
    if (err != MDBX_SUCCESS)
      ctx->error = err;
  }
  osal_condpair_unlock(&ctx->condpair);
  return ctx->error;
}

static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes, pgno_t pgno, pgno_t npages) {
  ASSERT(pgno == 0 || bytes > PAGEHDRSZ);
  while (bytes > 0) {
    const size_t side = ctx->head & 1;
    const size_t left = MDBX_ENVCOPY_WRITEBUF - ctx->write_len[side];
    if (left < (pgno ? PAGEHDRSZ : 1)) {
      int err = compacting_toggle_write_buffers(ctx);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      continue;
    }
    const size_t chunk = (bytes < left) ? bytes : left;
    void *const dst = ctx->write_buf[side] + ctx->write_len[side];
    if (src) {
      memcpy(dst, src, chunk);
      if (pgno) {
        ASSERT(chunk > PAGEHDRSZ);
        page_t *mp = dst;
        mp->pgno = pgno;
        if (mp->txnid == 0)
          mp->txnid = ctx->txn->txnid;
        if (mp->flags == P_LARGE) {
          ASSERT(bytes <= pgno2bytes(ctx->env, npages));
          mp->pages = npages;
        }
        pgno = 0;
      }
      src = ptr_disp(src, chunk);
    } else
      memset(dst, 0, chunk);
    bytes -= chunk;
    ctx->write_len[side] += chunk;
  }
  return MDBX_SUCCESS;
}

static int compacting_put_page(ctx_t *ctx, const page_t *mp, const size_t head_bytes, const size_t tail_bytes,
                               const pgno_t npages) {
  if (tail_bytes) {
    ASSERT(head_bytes + tail_bytes <= ctx->env->ps);
    ASSERT(npages == 1 && (page_type(mp) == P_BRANCH || page_type(mp) == P_LEAF));
  } else {
    ASSERT(head_bytes <= pgno2bytes(ctx->env, npages));
    ASSERT((npages == 1 && page_type(mp) == (P_LEAF | P_DUPFIX)) || page_type(mp) == P_LARGE);
  }

  const pgno_t pgno = ctx->first_unallocated;
  ctx->first_unallocated += npages;
  int err = compacting_put_bytes(ctx, mp, head_bytes, pgno, npages);
  if (unlikely(err != MDBX_SUCCESS))
    return err;
  err = compacting_put_bytes(ctx, nullptr, pgno2bytes(ctx->env, npages) - (head_bytes + tail_bytes), 0, 0);
  if (unlikely(err != MDBX_SUCCESS))
    return err;
  return compacting_put_bytes(ctx, ptr_disp(mp, ctx->env->ps - tail_bytes), tail_bytes, 0, 0);
}

__cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, pgno_t *const parent_pgno, txnid_t parent_txnid) {
  mc->top = 0;
  mc->ki[0] = 0;
  int rc = page_get(mc, *parent_pgno, &mc->pg[0], parent_txnid);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  rc = tree_deepen_edge(mc, Z_FIRST);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  /* Make cursor pages writable */
  const intptr_t deep_limit = mc->top + 1;
  void *const buf = osal_malloc(pgno2bytes(ctx->env, deep_limit + 1));
  if (buf == nullptr)
    return MDBX_ENOMEM;

  void *ptr = buf;
  for (intptr_t i = 0; i <= mc->top; i++) {
    page_copy(ptr, mc->pg[i], ctx->env->ps);
    mc->pg[i] = ptr;
    ptr = ptr_disp(ptr, ctx->env->ps);
  }
  /* This is writable space for a leaf page. Usually not needed. */
  page_t *const leaf = ptr;

  while (mc->top >= 0) {
    page_t *mp = mc->pg[mc->top];
    const size_t nkeys = page_numkeys(mp);
    if (is_leaf(mp)) {
      if (!(mc->flags & z_inner) /* may have nested N_TREE or N_BIG nodes */) {
        for (size_t i = 0; i < nkeys; i++) {
          node_t *node = page_node(mp, i);
          if (node_flags(node) == N_BIG) {
            /* Need writable leaf */
            if (mp != leaf) {
              mc->pg[mc->top] = leaf;
              page_copy(leaf, mp, ctx->env->ps);
              mp = leaf;
              node = page_node(mp, i);
            }

            const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid);
            if (unlikely((rc = lp.err) != MDBX_SUCCESS))
              goto bailout;
            const size_t datasize = node_ds(node);
            const pgno_t npages = largechunk_npages(ctx->env, datasize);
            poke_pgno(node_data(node), ctx->first_unallocated);
            rc = compacting_put_page(ctx, lp.page, PAGEHDRSZ + datasize, 0, npages);
            if (unlikely(rc != MDBX_SUCCESS))
              goto bailout;
          } else if (node_flags(node) & N_TREE) {
            if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) {
              ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size",
                    (unsigned)node_ds(node));
              rc = MDBX_CORRUPTED;
              goto bailout;
            }

            /* Need writable leaf */
            if (mp != leaf) {
              mc->pg[mc->top] = leaf;
              page_copy(leaf, mp, ctx->env->ps);
              mp = leaf;
              node = page_node(mp, i);
            }

            tree_t *nested = nullptr;
            if (node_flags(node) & N_DUP) {
              rc = cursor_dupsort_setup(mc, node, mp);
              if (likely(rc == MDBX_SUCCESS)) {
                nested = &mc->subcur->nested_tree;
                rc = compacting_walk(ctx, &mc->subcur->cursor, &nested->root, mp->txnid);
              }
            } else {
              cASSERT0(mc, (mc->flags & z_inner) == 0 && mc->subcur == 0);
              cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer);
              nested = &couple->inner.nested_tree;
              memcpy(nested, node_data(node), sizeof(tree_t));
              rc = compacting_walk_tree(ctx, nested);
            }
            if (unlikely(rc != MDBX_SUCCESS))
              goto bailout;
            memcpy(node_data(node), nested, sizeof(tree_t));
          }
        }
      }
    } else {
      mc->ki[mc->top]++;
      if (mc->ki[mc->top] < nkeys) {
        for (;;) {
          const node_t *node = page_node(mp, mc->ki[mc->top]);
          rc = page_get(mc, node_pgno(node), &mp, mp->txnid);
          if (unlikely(rc != MDBX_SUCCESS))
            goto bailout;
          mc->top += 1;
          if (unlikely(mc->top >= deep_limit)) {
            rc = MDBX_CURSOR_FULL;
            goto bailout;
          }
          mc->ki[mc->top] = 0;
          if (!is_branch(mp)) {
            mc->pg[mc->top] = mp;
            break;
          }
          /* Whenever we advance to a sibling branch page,
           * we must proceed all the way down to its first leaf. */
          page_copy(mc->pg[mc->top], mp, ctx->env->ps);
        }
        continue;
      }
    }

    const pgno_t pgno = ctx->first_unallocated;
    if (likely(!is_dupfix_leaf(mp))) {
      rc = compacting_put_page(ctx, mp, PAGEHDRSZ + mp->lower, ctx->env->ps - (PAGEHDRSZ + mp->upper), 1);
    } else {
      rc = compacting_put_page(ctx, mp, PAGEHDRSZ + page_numkeys(mp) * mp->dupfix_ksize, 0, 1);
    }
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;

    if (mc->top) {
      /* Update parent if there is one */
      node_set_pgno(page_node(mc->pg[mc->top - 1], mc->ki[mc->top - 1]), pgno);
      cursor_pop(mc);
    } else {
      /* Otherwise we're done */
      *parent_pgno = pgno;
      break;
    }
  }

bailout:
  osal_free(buf);
  return rc;
}

__cold static int compacting_walk_tree(ctx_t *ctx, tree_t *tree) {
  if (unlikely(tree->root == P_INVALID))
    return MDBX_SUCCESS; /* empty db */

  cursor_couple_t couple;
  memset(&couple, 0, sizeof(couple));
  couple.inner.cursor.signature = ~cur_signature_live;
  kvx_t kvx = {.clc = {.k = {.lmin = INT_MAX}, .v = {.lmin = INT_MAX}}};
  int rc = cursor_init4walk(&couple, ctx->txn, tree, &kvx);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  couple.outer.checking |= z_ignord | z_pagecheck;
  couple.inner.cursor.checking |= z_ignord | z_pagecheck;
  if (!tree->mod_txnid)
    tree->mod_txnid = ctx->txn->txnid;
  return compacting_walk(ctx, &couple.outer, &tree->root, tree->mod_txnid);
}

__cold static void compacting_fixup_meta(MDBX_env *env, meta_t *meta) {
  eASSERT0(env, meta->trees.gc.mod_txnid || meta->trees.gc.root == P_INVALID);
  eASSERT0(env, meta->trees.main.mod_txnid || meta->trees.main.root == P_INVALID);

  /* Calculate filesize taking in account shrink/growing thresholds */
  if (meta->geometry.first_unallocated != meta->geometry.now) {
    meta->geometry.now = meta->geometry.first_unallocated;
    const size_t aligner = pv2pages(meta->geometry.grow_pv ? meta->geometry.grow_pv : meta->geometry.shrink_pv);
    if (aligner) {
      const pgno_t aligned = pgno_ceil2os_pgno(env, meta->geometry.first_unallocated + aligner -
                                                        meta->geometry.first_unallocated % aligner);
      meta->geometry.now = aligned;
    }
  }

  if (meta->geometry.now < meta->geometry.lower)
    meta->geometry.now = meta->geometry.lower;
  if (meta->geometry.now > meta->geometry.upper)
    meta->geometry.now = meta->geometry.upper;

  /* Update signature */
  ASSERT(meta->geometry.now >= meta->geometry.first_unallocated);
  meta_sign_as_steady(meta);
}

/* Make resizable */
__cold static void meta_make_sizeable(meta_t *meta) {
  meta->geometry.lower = MIN_PAGENO;
  if (meta->geometry.grow_pv == 0) {
    const pgno_t step = 1 + (meta->geometry.upper - meta->geometry.lower) / 42;
    meta->geometry.grow_pv = pages2pv(step);
  }
  if (meta->geometry.shrink_pv == 0) {
    const pgno_t step = pv2pages(meta->geometry.grow_pv) << 1;
    meta->geometry.shrink_pv = pages2pv(step);
  }
}

__cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer,
                                       const bool dest_is_pipe, const MDBX_copy_flags_t flags) {
  const size_t meta_bytes = pgno2bytes(env, NUM_METAS);
  uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize);
  meta_t *const meta = meta_init_triplet(env, buffer);
  meta_set_txnid(env, meta, txn->txnid);

  if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
    meta_make_sizeable(meta);

  /* copy canary sequences if present */
  if (txn->canary.v) {
    meta->canary = txn->canary;
    meta->canary.v = constmeta_txnid(meta);
  }

  if (txn->dbs[MAIN_DBI].root == P_INVALID) {
    /* When the DB is empty, handle it specially to
     * fix any breakage like page leaks from ITS#8174. */
    meta->trees.main.flags = txn->dbs[MAIN_DBI].flags;
    compacting_fixup_meta(env, meta);
    if (dest_is_pipe) {
      if (flags & MDBX_CP_THROTTLE_MVCC)
        mdbx_txn_park(txn, false);
      int rc = osal_write(fd, buffer, meta_bytes);
      if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_THROTTLE_MVCC) != 0)
        rc = mdbx_txn_unpark(txn, false);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
    }
  } else {
    /* Count free pages + GC pages. */
    cursor_couple_t couple;
    int rc = cursor_init(&couple.outer, txn, FREE_DBI);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    pgno_t gc_npages = txn->dbs[FREE_DBI].branch_pages + txn->dbs[FREE_DBI].leaf_pages + txn->dbs[FREE_DBI].large_pages;
    MDBX_val key, data;
    rc = outer_first(&couple.outer, &key, &data);
    while (rc == MDBX_SUCCESS) {
      const pnl_t pnl = data.iov_base;
      if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < MDBX_PNL_SIZEOF(pnl))) {
        ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record length", data.iov_len);
        return MDBX_CORRUPTED;
      }
      if (unlikely(!pnl_check(pnl, txn->geo.first_unallocated))) {
        ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record content");
        return MDBX_CORRUPTED;
      }
      gc_npages += pnl_size(pnl);
      rc = outer_next(&couple.outer, &key, &data, MDBX_NEXT);
    }
    if (unlikely(rc != MDBX_NOTFOUND))
      return rc;

    meta->geometry.first_unallocated = txn->geo.first_unallocated - gc_npages;
    meta->trees.main = txn->dbs[MAIN_DBI];

    ctx_t ctx;
    memset(&ctx, 0, sizeof(ctx));
    rc = osal_condpair_init(&ctx.condpair);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;

    memset(data_buffer, 0, 2 * (size_t)MDBX_ENVCOPY_WRITEBUF);
    ctx.write_buf[0] = data_buffer;
    ctx.write_buf[1] = data_buffer + (size_t)MDBX_ENVCOPY_WRITEBUF;
    ctx.first_unallocated = NUM_METAS;
    ctx.env = env;
    ctx.fd = fd;
    ctx.txn = txn;
    ctx.flags = flags;

    osal_thread_t thread;
    int thread_err = osal_thread_create(&thread, compacting_write_thread, &ctx);
    if (likely(thread_err == MDBX_SUCCESS)) {
      if (dest_is_pipe) {
        if (!meta->trees.main.mod_txnid)
          meta->trees.main.mod_txnid = txn->txnid;
        compacting_fixup_meta(env, meta);
        if (flags & MDBX_CP_THROTTLE_MVCC)
          mdbx_txn_park(txn, false);
        rc = osal_write(fd, buffer, meta_bytes);
        if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_THROTTLE_MVCC) != 0)
          rc = mdbx_txn_unpark(txn, false);
      }
      if (likely(rc == MDBX_SUCCESS))
        rc = compacting_walk_tree(&ctx, &meta->trees.main);
      if (ctx.write_len[ctx.head & 1])
        /* toggle to flush non-empty buffers */
        compacting_toggle_write_buffers(&ctx);

      if (likely(rc == MDBX_SUCCESS) && unlikely(meta->geometry.first_unallocated != ctx.first_unallocated)) {
        if (ctx.first_unallocated > meta->geometry.first_unallocated) {
          ERROR("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO,
                "has double-used pages or other corruption", ctx.first_unallocated, '>',
                meta->geometry.first_unallocated);
          rc = MDBX_CORRUPTED; /* corrupted DB */
        }
        if (ctx.first_unallocated < meta->geometry.first_unallocated) {
          WARNING("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO,
                  "has page leak(s)", ctx.first_unallocated, '<', meta->geometry.first_unallocated);
          if (dest_is_pipe)
            /* the root within already written meta-pages is wrong */
            rc = MDBX_CORRUPTED;
        }
        /* fixup meta */
        meta->geometry.first_unallocated = ctx.first_unallocated;
      }

      /* toggle with empty buffers to exit thread's loop */
      eASSERT0(env, (ctx.write_len[ctx.head & 1]) == 0);
      compacting_toggle_write_buffers(&ctx);
      thread_err = osal_thread_join(thread);
      eASSERT0(env, (ctx.tail == ctx.head && ctx.write_len[ctx.head & 1] == 0) || ctx.error);
      osal_condpair_destroy(&ctx.condpair);
    }
    if (unlikely(thread_err != MDBX_SUCCESS))
      return thread_err;
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    if (unlikely(ctx.error != MDBX_SUCCESS))
      return ctx.error;
    if (!dest_is_pipe)
      compacting_fixup_meta(env, meta);
  }

  if (flags & MDBX_CP_THROTTLE_MVCC)
    mdbx_txn_park(txn, false);

  /* Extend file if required */
  if (meta->geometry.now != meta->geometry.first_unallocated) {
    const size_t whole_size = pgno2bytes(env, meta->geometry.now);
    if (!dest_is_pipe)
      return osal_fsetsize(fd, whole_size);

    const size_t used_size = pgno2bytes(env, meta->geometry.first_unallocated);
    memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF);
    for (size_t offset = used_size; offset < whole_size;) {
      const size_t chunk =
          ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset;
      int rc = osal_write(fd, data_buffer, chunk);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      offset += chunk;
    }
  }
  return MDBX_SUCCESS;
}

//----------------------------------------------------------------------------

__cold static int copy_asis(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer,
                            const bool dest_is_pipe, const MDBX_copy_flags_t flags) {
  const size_t meta_bytes = pgno2bytes(env, NUM_METAS);
  uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize);
  meta_t *const meta = meta_init_triplet(env, buffer);
  meta_set_txnid(env, meta, txn->txnid);

  if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
    meta_make_sizeable(meta);

  /* copy canary sequences if present */
  if (txn->canary.v) {
    meta->canary = txn->canary;
    meta->canary.v = constmeta_txnid(meta);
  }

  int rc = MDBX_SUCCESS;
  if (flags & MDBX_CP_THROTTLE_MVCC) {
    rc = mdbx_txn_park(txn, false);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }

  jitter4testing(false);
  size_t offset = meta_bytes;
  if (dest_is_pipe) {
    rc = osal_write(fd, buffer, meta_bytes);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    offset = 0;
  }

#if MDBX_USE_COPYFILERANGE
  static bool copyfilerange_unavailable;
#if (defined(__linux__) || defined(__gnu_linux__))
  if (globals.linux_kernel_version >= 0x05030000 && globals.linux_kernel_version < 0x05130000)
    copyfilerange_unavailable = true;
#endif /* linux */
  bool not_the_same_filesystem = false;
  if (!copyfilerange_unavailable) {
    struct statfs statfs_info;
    if (fstatfs(fd, &statfs_info) || statfs_info.f_type == /* ECRYPTFS_SUPER_MAGIC */ 0xf15f)
      /* avoid use copyfilerange_unavailable() to ecryptfs due bugs */
      not_the_same_filesystem = true;
  }
#endif /* MDBX_USE_COPYFILERANGE */

  /* Copy the data */
  const size_t whole_size = pgno_ceil2os_bytes(env, txn->geo.end_pgno);
  const size_t used_size = pgno2bytes(env, txn->geo.first_unallocated);
  while (rc == MDBX_SUCCESS && offset < used_size) {
    if (flags & MDBX_CP_THROTTLE_MVCC) {
      rc = mdbx_txn_unpark(txn, false);
      if (unlikely(rc != MDBX_SUCCESS))
        break;
    }

#if MDBX_USE_SENDFILE
    static bool sendfile_unavailable;
    if (dest_is_pipe && likely(!sendfile_unavailable)) {
      off_t in_offset = offset;
      const ssize_t written = sendfile(fd, env->lazy_fd, &in_offset, used_size - offset);
      if (likely(written > 0)) {
        offset = in_offset;
        if (flags & MDBX_CP_THROTTLE_MVCC)
          rc = mdbx_txn_park(txn, false);
        continue;
      }
      rc = MDBX_ENODATA;
      if (written == 0 || ignore_enosys_and_eagain(rc = errno) != MDBX_RESULT_TRUE)
        break;
      sendfile_unavailable = true;
    }
#endif /* MDBX_USE_SENDFILE */

#if MDBX_USE_COPYFILERANGE
    if (!dest_is_pipe && !not_the_same_filesystem && likely(!copyfilerange_unavailable)) {
      off_t in_offset = offset, out_offset = offset;
      ssize_t bytes_copied = copy_file_range(env->lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0);
      if (likely(bytes_copied > 0)) {
        offset = in_offset;
        if (flags & MDBX_CP_THROTTLE_MVCC)
          rc = mdbx_txn_park(txn, false);
        continue;
      }
      rc = MDBX_ENODATA;
      if (bytes_copied == 0)
        break;
      rc = errno;
      if (rc == EXDEV || rc == /* workaround for ecryptfs bug(s),
                                  maybe useful for others FS */
                             EINVAL)
        not_the_same_filesystem = true;
      else if (ignore_enosys_and_eagain(rc) == MDBX_RESULT_TRUE)
        copyfilerange_unavailable = true;
      else
        break;
    }
#endif /* MDBX_USE_COPYFILERANGE */

    /* fallback to portable */
    const size_t chunk =
        ((size_t)MDBX_ENVCOPY_WRITEBUF < used_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : used_size - offset;
    /* copy to avoid EFAULT in case swapped-out */
    memcpy(data_buffer, ptr_disp(env->dxb_mmap.base, offset), chunk);
    if (flags & MDBX_CP_THROTTLE_MVCC) {
      rc = mdbx_txn_park(txn, false);
      if (unlikely(rc != MDBX_SUCCESS))
        break;
    }
    rc = osal_write(fd, data_buffer, chunk);
    offset += chunk;
  }

  /* Extend file if required */
  if (likely(rc == MDBX_SUCCESS) && whole_size != used_size) {
    if (!dest_is_pipe)
      rc = osal_fsetsize(fd, whole_size);
    else {
      memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF);
      for (offset = used_size; rc == MDBX_SUCCESS && offset < whole_size;) {
        const size_t chunk =
            ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset;
        rc = osal_write(fd, data_buffer, chunk);
        offset += chunk;
      }
    }
  }

  return rc;
}

//----------------------------------------------------------------------------

__cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) {
  if (unlikely(txn->flags & MDBX_TXN_DIRTY))
    return MDBX_BAD_TXN;

  int rc = MDBX_SUCCESS;
  if (txn->flags & txn_ro_flat) {
    if (flags & MDBX_CP_THROTTLE_MVCC) {
      rc = mdbx_txn_park(txn, true);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
    }
  } else if (unlikely(flags & (MDBX_CP_THROTTLE_MVCC | MDBX_CP_RENEW_TXN)))
    return MDBX_EINVAL;

  const int dest_is_pipe = osal_is_pipe(fd);
  if (MDBX_IS_ERROR(dest_is_pipe))
    return dest_is_pipe;

  if (!dest_is_pipe) {
    rc = osal_fseek(fd, 0);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }

  MDBX_env *const env = txn->env;
  const size_t buffer_size =
      pgno_ceil2os_bytes(env, NUM_METAS) +
      ceil_powerof2(((flags & MDBX_CP_COMPACT) ? 2 * (size_t)MDBX_ENVCOPY_WRITEBUF : (size_t)MDBX_ENVCOPY_WRITEBUF),
                    globals.sys_pagesize);

  uint8_t *buffer = nullptr;
  rc = osal_memalign_alloc(globals.sys_pagesize, buffer_size, (void **)&buffer);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  if (!dest_is_pipe) {
    /* Firstly write a stub to meta-pages.
     * Now we sure to incomplete copy will not be used. */
    memset(buffer, -1, pgno2bytes(env, NUM_METAS));
    rc = osal_write(fd, buffer, pgno2bytes(env, NUM_METAS));
  }

  if (likely(rc == MDBX_SUCCESS))
    rc = mdbx_txn_unpark(txn, false);
  if (likely(rc == MDBX_SUCCESS)) {
    memset(buffer, 0, pgno2bytes(env, NUM_METAS));
    rc = ((flags & MDBX_CP_COMPACT) ? copy_with_compacting : copy_asis)(env, txn, fd, buffer, dest_is_pipe, flags);

    if (likely(rc == MDBX_SUCCESS))
      rc = mdbx_txn_unpark(txn, false);
  }

  if (txn->flags & txn_ro_flat) {
    if (flags & MDBX_CP_THROTTLE_MVCC)
      mdbx_txn_park(txn, true);
    else if (flags & MDBX_CP_DISPOSE_TXN)
      /* coverity[CHECKED_RETURN] */
      mdbx_txn_reset(txn);
  }

  if (!dest_is_pipe) {
    if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_DONT_FLUSH) == 0)
      rc = osal_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE);

    /* Write actual meta */
    if (likely(rc == MDBX_SUCCESS))
      rc = osal_pwrite(fd, buffer, pgno2bytes(env, NUM_METAS), 0);

    if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_DONT_FLUSH) == 0)
      rc = osal_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
  }

  osal_memalign_free(buffer);
  return rc;
}

__cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path, MDBX_copy_flags_t flags) {
  if (unlikely(!dest_path || *dest_path == '\0'))
    return MDBX_EINVAL;

  /* The destination path must exist, but the destination file must not.
   * We don't want the OS to cache the writes, since the source data is
   * already in the OS cache. */
  mdbx_filehandle_t newfd = INVALID_HANDLE_VALUE;
  int rc = osal_openfile((flags & MDBX_CP_OVERWRITE) ? MDBX_OPEN_COPY_OVERWRITE : MDBX_OPEN_COPY_EXCL, txn->env,
                         dest_path, &newfd,
#if defined(_WIN32) || defined(_WIN64)
                         (mdbx_mode_t)-1
#else
                         S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP
#endif
  );
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

#if defined(_WIN32) || defined(_WIN64)
  /* no locking required since the file opened with ShareMode == 0 */
#else
  MDBX_STRUCT_FLOCK lock_op;
  memset(&lock_op, 0, sizeof(lock_op));
  lock_op.l_type = F_WRLCK;
  lock_op.l_whence = SEEK_SET;
  lock_op.l_start = 0;
  lock_op.l_len = OFF_T_MAX;
  const int err_fcntl = MDBX_FCNTL(newfd, MDBX_F_SETLK, &lock_op) ? errno : MDBX_SUCCESS;

  const int err_flock =
#ifdef LOCK_EX
      flock(newfd, LOCK_EX | LOCK_NB) ? errno : MDBX_SUCCESS;
#else
      MDBX_ENOSYS;
#endif /* LOCK_EX */

  const int err_check_fs_local =
      /* avoid call osal_check_fs_local() on success */
      (!err_fcntl && !err_flock && MDBX_CHECKING < 1) ? MDBX_SUCCESS :
#if !defined(__ANDROID_API__) || __ANDROID_API__ >= 24
                                                      osal_check_fs_local(newfd, 0);
#else
                                                      MDBX_ENOSYS;
#endif

  const bool flock_may_fail =
#if defined(__linux__) || defined(__gnu_linux__)
      err_check_fs_local != 0;
#else
      true;
#endif /* Linux */

  if (!err_fcntl &&
      (err_flock == EWOULDBLOCK || err_flock == EAGAIN || ignore_enosys_and_eremote(err_flock) == MDBX_RESULT_TRUE)) {
    rc = err_flock;
    if (flock_may_fail) {
      WARNING("ignore %s(%" MDBX_PRIsPATH ") error %d: since %s done, local/remote-fs check %d", "flock", dest_path,
              err_flock, "fcntl-lock", err_check_fs_local);
      rc = MDBX_SUCCESS;
    }
  } else if (!err_flock && err_check_fs_local == MDBX_RESULT_TRUE &&
             ignore_enosys_and_eremote(err_fcntl) == MDBX_RESULT_TRUE) {
    WARNING("ignore %s(%" MDBX_PRIsPATH ") error %d: since %s done, local/remote-fs check %d", "fcntl-lock", dest_path,
            err_fcntl, "flock", err_check_fs_local);
  } else if (err_fcntl || err_flock) {
    ERROR("file-lock(%" MDBX_PRIsPATH ") failed: fcntl-lock %d, flock %d, local/remote-fs check %d", dest_path,
          err_fcntl, err_flock, err_check_fs_local);
    if (err_fcntl == ENOLCK || err_flock == ENOLCK)
      rc = ENOLCK;
    else if (err_fcntl == EWOULDBLOCK || err_flock == EWOULDBLOCK)
      rc = EWOULDBLOCK;
    else if (EWOULDBLOCK != EAGAIN && (err_fcntl == EAGAIN || err_flock == EAGAIN))
      rc = EAGAIN;
    else
      rc = (err_fcntl && ignore_enosys_and_eremote(err_fcntl) != MDBX_RESULT_TRUE) ? err_fcntl : err_flock;
  }
#endif /* Windows / POSIX */

  if (rc == MDBX_SUCCESS)
    rc = copy2fd(txn, newfd, flags);

  if (newfd != INVALID_HANDLE_VALUE) {
    int err = osal_closefile(newfd);
    if (rc == MDBX_SUCCESS && err != rc)
      rc = err;
    if (rc != MDBX_SUCCESS)
      (void)osal_removefile(dest_path);
  }
  return rc;
}

//----------------------------------------------------------------------------

__cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (likely(rc == MDBX_SUCCESS))
    rc = copy2fd(txn, fd, flags);
  if (flags & MDBX_CP_DISPOSE_TXN)
    mdbx_txn_abort(txn);
  return LOG_IFERR(rc);
}

__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) {
  if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN)))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  MDBX_txn *txn = nullptr;
  rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = copy2fd(txn, fd, flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN);
  mdbx_txn_abort(txn);
  return LOG_IFERR(rc);
}

__cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path, MDBX_copy_flags_t flags) {
#if defined(_WIN32) || defined(_WIN64)
  wchar_t *dest_pathW = nullptr;
  int rc = osal_mb2w(dest_path, &dest_pathW);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = mdbx_txn_copy2pathnameW(txn, dest_pathW, flags);
    osal_free(dest_pathW);
  }
  return LOG_IFERR(rc);
}

__cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path, MDBX_copy_flags_t flags) {
#endif /* Windows */
  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (likely(rc == MDBX_SUCCESS))
    rc = copy2pathname(txn, dest_path, flags);
  if (flags & MDBX_CP_DISPOSE_TXN)
    mdbx_txn_abort(txn);
  return LOG_IFERR(rc);
}

__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path, MDBX_copy_flags_t flags) {
#if defined(_WIN32) || defined(_WIN64)
  wchar_t *dest_pathW = nullptr;
  int rc = osal_mb2w(dest_path, &dest_pathW);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = mdbx_env_copyW(env, dest_pathW, flags);
    osal_free(dest_pathW);
  }
  return LOG_IFERR(rc);
}

__cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path, MDBX_copy_flags_t flags) {
#endif /* Windows */
  if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN)))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  MDBX_txn *txn = nullptr;
  rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = copy2pathname(txn, dest_path, flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN);
  mdbx_txn_abort(txn);
  return LOG_IFERR(rc);
}

MDBX_cursor *mdbx_cursor_create(void *context) {
  cursor_couple_t *couple = osal_calloc(1, sizeof(cursor_couple_t));
  if (unlikely(!couple))
    return nullptr;

  VALGRIND_MAKE_MEM_UNDEFINED(couple, sizeof(cursor_couple_t));
  couple->outer.signature = cur_signature_ready4dispose;
  couple->outer.next = &couple->outer;
  couple->userctx = context;
  cursor_reset(couple);
  VALGRIND_MAKE_MEM_DEFINED(&couple->outer.backup, sizeof(couple->outer.backup));
  VALGRIND_MAKE_MEM_DEFINED(&couple->outer.tree, sizeof(couple->outer.tree));
  VALGRIND_MAKE_MEM_DEFINED(&couple->outer.clc, sizeof(couple->outer.clc));
  VALGRIND_MAKE_MEM_DEFINED(&couple->outer.dbi_state, sizeof(couple->outer.dbi_state));
  VALGRIND_MAKE_MEM_DEFINED(&couple->outer.subcur, sizeof(couple->outer.subcur));
  VALGRIND_MAKE_MEM_DEFINED(&couple->outer.txn, sizeof(couple->outer.txn));
  return &couple->outer;
}

int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *mc) {
  return likely(mc) ? mdbx_cursor_bind(txn, mc, (kvx_t *)mc->clc - txn->env->kvs) : LOG_IFERR(MDBX_EINVAL);
}

int mdbx_cursor_reset(MDBX_cursor *mc) {
  int rc = cursor_check(mc, MDBX_TXN_FINISHED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_reset((cursor_couple_t *)mc);
  return MDBX_SUCCESS;
}

int mdbx_cursor_bind(MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) {
  if (unlikely(!mc))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live)) {
    int rc = (mc->signature == cur_signature_wait4eot) ? MDBX_EINVAL : MDBX_EBADSIGN;
    return LOG_IFERR(rc);
  }

  int rc = check_txn(txn, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(dbi == FREE_DBI) && ((txn->flags & txn_ro_both) == 0 || txn->parent))
    return LOG_IFERR(MDBX_EACCESS);

  rc = dbi_check(txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(mc->backup)) /* Cursor from parent transaction */
    LOG_IFERR(MDBX_EINVAL);

  if (mc->signature == cur_signature_live) {
    if (mc->txn == txn && cursor_dbi(mc) == dbi)
      return MDBX_SUCCESS;
    rc = mdbx_cursor_unbind(mc);
    if (unlikely(rc != MDBX_SUCCESS))
      return (rc == MDBX_BAD_TXN) ? MDBX_EINVAL : rc;
  }
  cASSERT0(mc, mc->next == mc);

  rc = cursor_init(mc, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  mc->next = txn->cursors[dbi];
  txn->cursors[dbi] = mc;
  txn->flags |= txn_may_have_cursors;
  return MDBX_SUCCESS;
}

int mdbx_cursor_unbind(MDBX_cursor *mc) {
  if (unlikely(!mc))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(mc->signature != cur_signature_live))
    return (mc->signature == cur_signature_ready4dispose) ? MDBX_SUCCESS : LOG_IFERR(MDBX_EBADSIGN);

  if (unlikely(mc->backup)) /* Cursor from parent transaction */
    /* TODO: реализовать при переходе на двусвязный список курсоров */
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(mc->txn, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD);
  if (unlikely(rc != MDBX_SUCCESS)) {
    for (const MDBX_txn *txn = mc->txn; rc == MDBX_BAD_TXN && check_txn(txn, MDBX_TXN_FINISHED) == MDBX_SUCCESS;
         txn = txn->nested)
      if (dbi_state(txn, cursor_dbi(mc)) == 0)
        /* специальный случай: курсор прикреплён к родительской транзакции, но соответствующий dbi-дескриптор ещё
         * не использовался во вложенной транзакции, т.е. курсор ещё не импортирован в дочернюю транзакцию и не имеет
         * связанного сохранённого состояния (поэтому mc→backup равен nullptr). */
        rc = MDBX_EINVAL;
    return LOG_IFERR(rc);
  }

  if (unlikely(!mc->txn || mc->txn->signature != txn_signature)) {
    ERROR("Wrong cursor's transaction %p 0x%x", __Wpedantic_format_voidptr(mc->txn), mc->txn ? mc->txn->signature : 0);
    return LOG_IFERR(MDBX_PROBLEM);
  }

  if (mc->next != mc) {
    const size_t dbi = cursor_dbi(mc);
    cASSERT0(mc, dbi < mc->txn->n_dbi);
    cASSERT0(mc, &mc->txn->env->kvs[dbi].clc == mc->clc);
    if (dbi < mc->txn->n_dbi) {
      MDBX_cursor **prev = &mc->txn->cursors[dbi];
      while (/* *prev && */ *prev != mc) {
        ENSURE_OBJ(mc, (*prev)->signature == cur_signature_live || (*prev)->signature == cur_signature_wait4eot);
        prev = &(*prev)->next;
      }
      cASSERT0(mc, *prev == mc);
      *prev = mc->next;
    }
    mc->next = mc;
  }
  cursor_drown((cursor_couple_t *)mc);
  mc->signature = cur_signature_ready4dispose;
  return MDBX_SUCCESS;
}

int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) {
  if (unlikely(!ret))
    return LOG_IFERR(MDBX_EINVAL);
  *ret = nullptr;

  MDBX_cursor *const mc = mdbx_cursor_create(nullptr);
  if (unlikely(!mc))
    return LOG_IFERR(MDBX_ENOMEM);

  int rc = mdbx_cursor_bind(txn, mc, dbi);
  if (unlikely(rc != MDBX_SUCCESS)) {
    mdbx_cursor_close(mc);
    return LOG_IFERR(rc);
  }

  *ret = mc;
  return MDBX_SUCCESS;
}

void mdbx_cursor_close(MDBX_cursor *cursor) {
  if (likely(cursor)) {
    int err = mdbx_cursor_close2(cursor);
    if (unlikely(err != MDBX_SUCCESS))
      panic_fmt(cursor, "error %d (%s) while closing cursor", err, mdbx_liberr2str(err));
  }
}

int mdbx_cursor_close2(MDBX_cursor *mc) {
  if (unlikely(!mc))
    return LOG_IFERR(MDBX_EINVAL);

  if (mc->signature == cur_signature_ready4dispose) {
    if (unlikely(mc->txn || mc->backup))
      return LOG_IFERR(MDBX_PANIC);
    cursor_drown((cursor_couple_t *)mc);
    mc->signature = 0;
    osal_free(mc);
    return MDBX_SUCCESS;
  }

  if (unlikely(mc->signature != cur_signature_live))
    return LOG_IFERR(MDBX_EBADSIGN);

  MDBX_txn *const txn = mc->txn;
  int rc = check_txn(txn, MDBX_TXN_FINISHED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (mc->backup) {
    /* Cursor closed before nested txn ends */
    cursor_reset((cursor_couple_t *)mc);
    mc->signature = cur_signature_wait4eot;
    return MDBX_SUCCESS;
  }

  if (mc->next != mc) {
    const size_t dbi = cursor_dbi(mc);
    cASSERT0(mc, dbi < mc->txn->n_dbi);
    cASSERT0(mc, &mc->txn->env->kvs[dbi].clc == mc->clc);
    if (likely(dbi < txn->n_dbi)) {
      MDBX_cursor **prev = &txn->cursors[dbi];
      while (/* *prev && */ *prev != mc) {
        ENSURE_OBJ(txn, (*prev)->signature == cur_signature_live || (*prev)->signature == cur_signature_wait4eot);
        prev = &(*prev)->next;
      }
      cASSERT0(txn, *prev == mc);
      *prev = mc->next;
    }
    mc->next = mc;
  }
  cursor_drown((cursor_couple_t *)mc);
  mc->signature = 0;
  osal_free(mc);
  return MDBX_SUCCESS;
}

int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest) {
  if (unlikely(src == dest))
    return LOG_IFERR(MDBX_EINVAL);
  int rc = cursor_check(src, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = mdbx_cursor_bind(src->txn, dest, cursor_dbi(src));
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  cursor_cpstk(src, dest);
  if (src->subcur) {
    dest->subcur->nested_tree = src->subcur->nested_tree;
    dest->subcur->cursor.tree = &dest->subcur->nested_tree;
    cursor_cpstk(&src->subcur->cursor, &dest->subcur->cursor);
  }
  return MDBX_SUCCESS;
}

int mdbx_txn_release_all_cursors_ex(const MDBX_txn *txn, bool unbind, size_t *count) {
  int rc = check_txn(txn, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  size_t n = 0;
  do {
    TXN_FOREACH_DBI_FROM(txn, i, MAIN_DBI) {
      MDBX_cursor *mc = txn->cursors[i], *next = nullptr;
      if (mc) {
        txn->cursors[i] = nullptr;
        do {
          next = mc->next;
          if (mc->signature == cur_signature_live) {
            mc->signature = cur_signature_wait4eot;
            cursor_drown((cursor_couple_t *)mc);
          } else
            ENSURE(mc->signature == cur_signature_wait4eot);
          if (mc->backup) {
            MDBX_cursor *bk = mc->backup;
            mc->next = bk->next;
            mc->backup = bk->backup;
            bk->backup = nullptr;
            bk->signature = 0;
            osal_free(bk);
          } else {
            mc->signature = cur_signature_ready4dispose;
            mc->next = mc;
            ++n;
            if (!unbind) {
              mc->signature = 0;
              osal_free(mc);
            }
          }
        } while ((mc = next) != nullptr);
      }
    }
    txn = txn->parent;
  } while (txn);

  if (count)
    *count = n;
  return MDBX_SUCCESS;
}

int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r, bool ignore_multival) {
  const int incomparable = INT16_MAX + 1;

  if (unlikely(!l))
    return r ? -incomparable * 9 : 0;
  else if (unlikely(!r))
    return incomparable * 9;

  if (unlikely(cursor_check_pure(l) != MDBX_SUCCESS))
    return (cursor_check_pure(r) == MDBX_SUCCESS) ? -incomparable * 8 : 0;
  if (unlikely(cursor_check_pure(r) != MDBX_SUCCESS))
    return (cursor_check_pure(l) == MDBX_SUCCESS) ? incomparable * 8 : 0;

  if (unlikely(l->clc != r->clc)) {
    if (l->txn->env != r->txn->env)
      return (l->txn->env > r->txn->env) ? incomparable * 7 : -incomparable * 7;
    if (l->txn->txnid != r->txn->txnid)
      return (l->txn->txnid > r->txn->txnid) ? incomparable * 6 : -incomparable * 6;
    return (l->clc > r->clc) ? incomparable * 5 : -incomparable * 5;
  }
  ASSERT(cursor_dbi(l) == cursor_dbi(r));

  int diff = is_pointed(l) - is_pointed(r);
  if (unlikely(diff))
    return (diff > 0) ? incomparable * 4 : -incomparable * 4;
  if (unlikely(!is_pointed(l)))
    return 0;

  intptr_t detent = (l->top <= r->top) ? l->top : r->top;
  for (intptr_t i = 0; i <= detent; ++i) {
    diff = l->ki[i] - r->ki[i];
    if (diff)
      return diff;
  }
  if (unlikely(l->top != r->top))
    return (l->top > r->top) ? incomparable * 3 : -incomparable * 3;

  ASSERT((l->subcur != nullptr) == (r->subcur != nullptr));
  if (unlikely((l->subcur != nullptr) != (r->subcur != nullptr)))
    return l->subcur ? incomparable * 2 : -incomparable * 2;
  if (ignore_multival || !l->subcur)
    return 0;

  if (is_pointed(&l->subcur->cursor)) {
    const page_t *mp = l->pg[l->top];
    const node_t *node = page_node(mp, l->ki[l->top]);
    ASSERT(node_flags(node) & N_DUP);
  }
  if (is_pointed(&r->subcur->cursor)) {
    const page_t *mp = r->pg[r->top];
    const node_t *node = page_node(mp, r->ki[r->top]);
    ASSERT(node_flags(node) & N_DUP);
  }

  l = &l->subcur->cursor;
  r = &r->subcur->cursor;
  diff = is_pointed(l) - is_pointed(r);
  if (unlikely(diff))
    return (diff > 0) ? incomparable * 2 : -incomparable * 2;
  if (unlikely(!is_pointed(l)))
    return 0;

  detent = (l->top <= r->top) ? l->top : r->top;
  for (intptr_t i = 0; i <= detent; ++i) {
    diff = l->ki[i] - r->ki[i];
    if (diff)
      return diff;
  }
  if (unlikely(l->top != r->top))
    return (l->top > r->top) ? incomparable : -incomparable;

  return (l->flags & z_eof_hard) - (r->flags & z_eof_hard);
}

int mdbx_cursor_count_ex(const MDBX_cursor *mc, size_t *count, MDBX_stat *ns, size_t bytes) {
  int rc = cursor_check_ro(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (ns) {
    const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid);
    if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid)
      return LOG_IFERR(MDBX_EINVAL);
    memset(ns, 0, sizeof(*ns));
  }

  size_t nvals = 0;
  if (is_filled(mc)) {
    nvals = 1;
    if (!inner_hollow(mc)) {
      const page_t *mp = mc->pg[mc->top];
      const node_t *node = page_node(mp, mc->ki[mc->top]);
      cASSERT0(mc, node_flags(node) & N_DUP);
      const tree_t *nt = &mc->subcur->nested_tree;
      nvals = unlikely(nt->items > PTRDIFF_MAX) ? PTRDIFF_MAX : (size_t)nt->items;
      if (ns) {
        ns->ms_psize = (unsigned)node_ds(node);
        if (node_flags(node) & N_TREE) {
          ns->ms_psize = mc->txn->env->ps;
          ns->ms_depth = nt->height;
          ns->ms_branch_pages = nt->branch_pages;
        }
        cASSERT0(mc, nt->large_pages == 0);
        ns->ms_leaf_pages = nt->leaf_pages;
        ns->ms_entries = nt->items;
        if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(ns->ms_mod_txnid)))
          ns->ms_mod_txnid = nt->mod_txnid;
      }
    }
  }

  if (likely(count))
    *count = nvals;

  return MDBX_SUCCESS;
}

int mdbx_cursor_count(const MDBX_cursor *mc, size_t *count) {
  if (unlikely(count == nullptr))
    return LOG_IFERR(MDBX_EINVAL);

  return mdbx_cursor_count_ex(mc, count, nullptr, 0);
}

int mdbx_cursor_on_first(const MDBX_cursor *mc) {
  int rc = cursor_check_pure(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return cursor_on_first(mc);
}

int mdbx_cursor_on_first_dup(const MDBX_cursor *mc) {
  int rc = cursor_check_pure(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return (is_filled(mc) && mc->subcur) ? cursor_on_first(&mc->subcur->cursor) : MDBX_RESULT_TRUE;
}

int mdbx_cursor_on_last(const MDBX_cursor *mc) {
  int rc = cursor_check_pure(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return cursor_on_last(mc);
}

int mdbx_cursor_on_last_dup(const MDBX_cursor *mc) {
  int rc = cursor_check_pure(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return (is_filled(mc) && mc->subcur) ? cursor_on_last(&mc->subcur->cursor) : MDBX_RESULT_TRUE;
}

int mdbx_cursor_eof(const MDBX_cursor *mc) {
  int rc = cursor_check_pure(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return is_eof(mc) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
}

int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
  int rc = cursor_check_ro(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return LOG_IFERR(cursor_ops(mc, key, data, op));
}

__hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func predicate, void *context, void *arg, MDBX_val *key,
                               MDBX_val *value, MDBX_cursor_op turn_op) {
  int rc;
  switch (turn_op) {
  case MDBX_NEXT:
  case MDBX_NEXT_NODUP:
    for (;;) {
      rc = predicate(context, key, value, arg);
      if (rc != MDBX_RESULT_FALSE)
        return rc;
      rc = outer_next(mc, key, value, turn_op);
      if (unlikely(rc != MDBX_SUCCESS))
        return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc;
    }

  case MDBX_PREV:
  case MDBX_PREV_NODUP:
    for (;;) {
      rc = predicate(context, key, value, arg);
      if (rc != MDBX_RESULT_FALSE)
        return rc;
      rc = outer_prev(mc, key, value, turn_op);
      if (unlikely(rc != MDBX_SUCCESS))
        return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc;
    }

  case MDBX_NEXT_DUP:
    if (mc->subcur)
      for (;;) {
        rc = predicate(context, key, value, arg);
        if (rc != MDBX_RESULT_FALSE)
          return rc;
        rc = inner_next(&mc->subcur->cursor, value);
        if (unlikely(rc != MDBX_SUCCESS))
          return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc;
      }
    return MDBX_NOTFOUND;

  case MDBX_PREV_DUP:
    if (mc->subcur)
      for (;;) {
        rc = predicate(context, key, value, arg);
        if (rc != MDBX_RESULT_FALSE)
          return rc;
        rc = inner_prev(&mc->subcur->cursor, value);
        if (unlikely(rc != MDBX_SUCCESS))
          return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc;
      }
    return MDBX_NOTFOUND;

  default:
    for (;;) {
      rc = predicate(context, key, value, arg);
      if (rc != MDBX_RESULT_FALSE)
        return rc;
      rc = cursor_ops(mc, key, value, turn_op);
      if (unlikely(rc != MDBX_SUCCESS))
        return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc;
    }
  }
}

int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func predicate, void *context, MDBX_cursor_op start_op,
                     MDBX_cursor_op turn_op, void *arg) {
  if (unlikely(!predicate))
    return LOG_IFERR(MDBX_EINVAL);

  const unsigned valid_start_mask = 1 << MDBX_FIRST | 1 << MDBX_FIRST_DUP | 1 << MDBX_LAST | 1 << MDBX_LAST_DUP |
                                    1 << MDBX_GET_CURRENT | 1 << MDBX_GET_MULTIPLE;
  if (unlikely(start_op > 30 || ((1 << start_op) & valid_start_mask) == 0))
    return LOG_IFERR(MDBX_EINVAL);

  const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV |
                                   1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE |
                                   1 << MDBX_PREV_MULTIPLE;
  if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0))
    return LOG_IFERR(MDBX_EINVAL);

  MDBX_val key = {nullptr, 0}, value = {nullptr, 0};
  int rc = mdbx_cursor_get(mc, &key, &value, start_op);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  return LOG_IFERR(scan_confinue(mc, predicate, context, arg, &key, &value, turn_op));
}

int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func predicate, void *context, MDBX_cursor_op from_op,
                          MDBX_val *key, MDBX_val *value, MDBX_cursor_op turn_op, void *arg) {
  if (unlikely(!predicate || !key))
    return LOG_IFERR(MDBX_EINVAL);

  const unsigned valid_start_mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY |
                                    1 << MDBX_GET_MULTIPLE | 1 << MDBX_SET_LOWERBOUND | 1 << MDBX_SET_UPPERBOUND;
  if (unlikely(from_op < MDBX_TO_KEY_LESSER_THAN && ((1 << from_op) & valid_start_mask) == 0))
    return LOG_IFERR(MDBX_EINVAL);

  const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV |
                                   1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE |
                                   1 << MDBX_PREV_MULTIPLE;
  if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = mdbx_cursor_get(mc, key, value, from_op);
  if (unlikely(MDBX_IS_ERROR(rc)))
    return LOG_IFERR(rc);

  cASSERT0(mc, key != nullptr);
  MDBX_val stub;
  if (!value) {
    value = &stub;
    rc = cursor_ops(mc, key, value, MDBX_GET_CURRENT);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }
  return LOG_IFERR(scan_confinue(mc, predicate, context, arg, key, value, turn_op));
}

int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, size_t limit, MDBX_cursor_op op) {
  if (unlikely(!count))
    return LOG_IFERR(MDBX_EINVAL);

  *count = 0;
  if (unlikely(limit < 4 || limit > INTPTR_MAX - 2))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = cursor_check_ro(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(mc->subcur))
    return LOG_IFERR(MDBX_INCOMPATIBLE) /* must be a non-dupsort table */;

  switch (op) {
  case MDBX_NEXT:
    if (unlikely(is_eof(mc)))
      return LOG_IFERR(is_pointed(mc) ? MDBX_NOTFOUND : MDBX_ENODATA);
    break;

  case MDBX_FIRST:
    if (!is_filled(mc)) {
      rc = outer_first(mc, nullptr, nullptr);
      if (unlikely(rc != MDBX_SUCCESS))
        return LOG_IFERR(rc);
    }
    break;

  default:
    DEBUG("unhandled/unimplemented cursor operation %u", op);
    return LOG_IFERR(MDBX_EINVAL);
  }

  const page_t *mp = mc->pg[mc->top];
  size_t nkeys = page_numkeys(mp);
  size_t ki = mc->ki[mc->top];
  size_t n = 0;
  while (n + 2 <= limit) {
    cASSERT0(mc, ki < nkeys);
    if (unlikely(ki >= nkeys))
      goto sibling;

    const node_t *leaf = page_node(mp, ki);
    pairs[n] = get_key(leaf);
    rc = node_read(mc, leaf, &pairs[n + 1], mp);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;

    n += 2;
    if (++ki == nkeys) {
    sibling:
      rc = cursor_sibling_right(mc);
      if (rc != MDBX_SUCCESS) {
        if (rc == MDBX_NOTFOUND)
          rc = MDBX_RESULT_TRUE;
        goto bailout;
      }

      mp = mc->pg[mc->top];
      DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]);
      if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
        ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
        rc = MDBX_CORRUPTED;
        goto bailout;
      }
      nkeys = page_numkeys(mp);
      ki = 0;
    }
  }
  mc->ki[mc->top] = (indx_t)ki;

bailout:
  *count = n;
  return LOG_IFERR(rc);
}

/*----------------------------------------------------------------------------*/

int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) {
  int rc = cursor_check(mc, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer);
  couple->userctx = ctx;
  return MDBX_SUCCESS;
}

void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) {
  if (unlikely(!mc))
    return nullptr;

  if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live))
    return nullptr;

  cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer);
  return couple->userctx;
}

MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *mc) {
  if (unlikely(!mc || mc->signature != cur_signature_live))
    return nullptr;
  MDBX_txn *txn = mc->txn;
  if (unlikely(!txn || txn->signature != txn_signature || (txn->flags & MDBX_TXN_FINISHED)))
    return nullptr;
  return (txn->flags & MDBX_TXN_HAS_CHILD) ? txn->env->txn : txn;
}

MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) {
  if (unlikely(!mc || mc->signature != cur_signature_live))
    return UINT_MAX;
  return cursor_dbi(mc);
}

/*----------------------------------------------------------------------------*/

int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) {
  if (unlikely(key == nullptr || data == nullptr))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = cursor_check_rw(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(flags & MDBX_MULTIPLE)) {
    rc = cursor_check_multiple(mc, key, data, flags);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  if (flags & MDBX_RESERVE) {
    if (unlikely(mc->tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_INTEGERDUP | MDBX_DUPFIXED)))
      return LOG_IFERR(MDBX_INCOMPATIBLE);
    data->iov_base = nullptr;
  }

  return LOG_IFERR(cursor_put_checklen(mc, key, data, flags));
}

int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) {
  int rc = cursor_check_rw(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return LOG_IFERR(cursor_del(mc, flags));
}

__cold int mdbx_cursor_ignord(MDBX_cursor *mc) {
  int rc = cursor_check(mc, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  mc->checking |= z_ignord;
  if (mc->subcur)
    mc->subcur->cursor.checking |= z_ignord;

  return MDBX_SUCCESS;
}

/*----------------------------------------------------------------------------*/

#if MDBX_ENABLE_BUNCHES_REMOVAL
static MDBX_cursor *get_axe(MDBX_cursor *mc, cursor_couple_t *couple) {
  MDBX_cursor *axe = cursor_clone_complete(mc, couple);
  axe->next = axe->txn->cursors[cursor_dbi(axe)];
  axe->txn->cursors[cursor_dbi(axe)] = axe;
  return axe;
}
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */

int mdbx_cursor_bunch_delete(MDBX_cursor *mc, MDBX_bunch_action_t action, uint64_t *number_of_affected) {
  int rc = cursor_check_rw(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(action < MDBX_DELETE_CURRENT_VALUE || action > MDBX_DELETE_WHOLE))
    return LOG_IFERR(MDBX_EINVAL);

#if MDBX_ENABLE_BUNCHES_REMOVAL
  cursor_couple_t couple;
  MDBX_cursor *axe = nullptr;
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */

  const uint64_t save_items = mc->tree->items;
  switch (action) {
  default:
    rc = MDBX_EINVAL;
    break;
  case MDBX_DELETE_CURRENT_VALUE:
    rc = cursor_del(mc, 0);
    break;
  case MDBX_DELETE_CURRENT_MULTIVAL_ALL:
    rc = cursor_del(mc, MDBX_ALLDUPS);
    break;
  case MDBX_DELETE_WHOLE:
    rc = tbl_purge(mc);
    break;

#if MDBX_ENABLE_BUNCHES_REMOVAL
  case MDBX_DELETE_CURRENT_MULTIVAL_BEFORE_EXCLUDING:
  case MDBX_DELETE_CURRENT_MULTIVAL_BEFORE_INCLUDING:
    axe = get_axe(mc, &couple);
    if (inner_pointed(axe)) {
      rc = inner_first(&axe->subcur->cursor, nullptr);
      if (rc != MDBX_SUCCESS) {
        rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
        break;
      }
    }
    rc = tree_curoff_range(axe, mc, action == MDBX_DELETE_CURRENT_MULTIVAL_BEFORE_INCLUDING);
#else
  case MDBX_DELETE_CURRENT_MULTIVAL_BEFORE_INCLUDING:
    rc = cursor_del(mc, 0);
    __fallthrough /* fall through */;
  case MDBX_DELETE_CURRENT_MULTIVAL_BEFORE_EXCLUDING:
    while (rc == MDBX_SUCCESS) {
      rc = outer_prev(mc, nullptr, nullptr, MDBX_PREV_DUP);
      if (rc != MDBX_SUCCESS) {
        rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
        break;
      }
      rc = cursor_del(mc, 0);
    }
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
    break;

#if MDBX_ENABLE_BUNCHES_REMOVAL
  case MDBX_DELETE_CURRENT_MULTIVAL_AFTER_EXCLUDING:
    rc = outer_next(mc, nullptr, nullptr, MDBX_NEXT_DUP);
    if (rc != MDBX_SUCCESS) {
      rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
      break;
    }
    __fallthrough /* fall through */;
  case MDBX_DELETE_CURRENT_MULTIVAL_AFTER_INCLUDING:
    axe = get_axe(mc, &couple);
    if (inner_pointed(mc)) {
      rc = inner_last(&mc->subcur->cursor, nullptr);
      if (rc != MDBX_SUCCESS) {
        rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
        break;
      }
    }
    rc = tree_curoff_range(axe, mc, true);
#else
  case MDBX_DELETE_CURRENT_MULTIVAL_AFTER_INCLUDING:
    rc = cursor_del(mc, 0);
    __fallthrough /* fall through */;
  case MDBX_DELETE_CURRENT_MULTIVAL_AFTER_EXCLUDING:
    while (rc == MDBX_SUCCESS) {
      rc = outer_next(mc, nullptr, nullptr, MDBX_NEXT_DUP);
      mc->flags &= ~z_eof_hard;
      ((cursor_couple_t *)mc)->inner.cursor.flags &= ~z_eof_hard;
      if (rc != MDBX_SUCCESS) {
        rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
        break;
      }
      rc = cursor_del(mc, 0);
    }
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
    break;

#if MDBX_ENABLE_BUNCHES_REMOVAL
  case MDBX_DELETE_BEFORE_INCLUDING:
  case MDBX_DELETE_BEFORE_EXCLUDING:
    axe = get_axe(mc, &couple);
    rc = outer_first(axe, nullptr, nullptr);
    if (rc != MDBX_SUCCESS) {
      rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
      break;
    }
    rc = tree_curoff_range(axe, mc, action == MDBX_DELETE_BEFORE_INCLUDING);
#else
  case MDBX_DELETE_BEFORE_INCLUDING:
    rc = cursor_del(mc, 0);
    __fallthrough /* fall through */;
  case MDBX_DELETE_BEFORE_EXCLUDING:
    while (rc == MDBX_SUCCESS) {
      rc = outer_prev(mc, nullptr, nullptr, MDBX_PREV);
      if (rc != MDBX_SUCCESS) {
        rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
        break;
      }
      rc = cursor_del(mc, 0);
    }
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
    break;

#if MDBX_ENABLE_BUNCHES_REMOVAL
  case MDBX_DELETE_AFTER_EXCLUDING:
    rc = outer_next(mc, nullptr, nullptr, MDBX_NEXT);
    if (rc != MDBX_SUCCESS) {
      rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
      break;
    }
    __fallthrough /* fall through */;
  case MDBX_DELETE_AFTER_INCLUDING:
    axe = get_axe(mc, &couple);
    rc = outer_last(mc, nullptr, nullptr);
    if (rc != MDBX_SUCCESS) {
      rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
      break;
    }
    rc = tree_curoff_range(axe, mc, true);
#else
  case MDBX_DELETE_AFTER_INCLUDING:
    rc = cursor_del(mc, 0);
    __fallthrough /* fall through */;
  case MDBX_DELETE_AFTER_EXCLUDING:
    while (rc == MDBX_SUCCESS) {
      rc = outer_next(mc, nullptr, nullptr, MDBX_NEXT);
      mc->flags &= ~z_eof_hard;
      ((cursor_couple_t *)mc)->inner.cursor.flags &= ~z_eof_hard;
      if (rc != MDBX_SUCCESS) {
        rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
        break;
      }
      rc = cursor_del(mc, 0);
    }
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
    break;
  }

#if MDBX_ENABLE_BUNCHES_REMOVAL
  if (axe)
    axe->txn->cursors[cursor_dbi(axe)] = axe->next;
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
  if (number_of_affected)
    *number_of_affected = save_items - mc->tree->items;

  return LOG_IFERR(rc);
}

int mdbx_cursor_delete_range(MDBX_cursor *begin, MDBX_cursor *end, bool end_including, uint64_t *number_of_affected) {
  if (number_of_affected)
    *number_of_affected = 0;
  if (unlikely(!begin && !end))
    return LOG_IFERR(MDBX_EINVAL);

  int rc;
  if (begin) {
    rc = cursor_check_rw(begin);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    if (unlikely(!is_pointed(begin)))
      return LOG_IFERR(MDBX_ENODATA);
  }

  if (end) {
    rc = cursor_check_rw(end);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    if (unlikely(!is_pointed(end)))
      return LOG_IFERR(MDBX_ENODATA);
  }

  if (unlikely(begin && end && begin->txn != end->txn && begin->tree != end->tree))
    return LOG_IFERR(MDBX_EINVAL);

  cursor_couple_t couple;
  if (!begin) {
    begin = cursor_clone_complete(end, &couple);
    rc = outer_first(begin, nullptr, nullptr);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    couple.outer.next = begin->txn->cursors[cursor_dbi(begin)];
    begin->txn->cursors[cursor_dbi(begin)] = &couple.outer;
  }

  if (!end) {
    end = cursor_clone_complete(begin, &couple);
    rc = outer_last(end, nullptr, nullptr);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    couple.outer.next = end->txn->cursors[cursor_dbi(end)];
    end->txn->cursors[cursor_dbi(end)] = &couple.outer;
  }

  const uint64_t save_items = begin->tree->items;
  rc = tree_curoff_range(begin, end, !!end_including);
  if (number_of_affected)
    *number_of_affected = save_items - begin->tree->items;

  if (begin->txn->cursors[cursor_dbi(begin)] == &couple.outer)
    /* coverity[UNINIT] */
    couple.outer.txn->cursors[cursor_dbi(begin)] = couple.outer.next;

  return LOG_IFERR(rc);
}

int mdbx_cursor_distribute(const MDBX_cursor *begin, const MDBX_cursor *end, MDBX_cursor **array, intptr_t array_size,
                           unsigned deepness) {
  if (unlikely(!begin && !end))
    return LOG_IFERR(MDBX_EINVAL);
  if (unlikely(!array || array_size < 1))
    return LOG_IFERR(MDBX_EINVAL);

  int rc;
  if (begin) {
    rc = cursor_check_ro(begin);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    if (unlikely(!is_pointed(begin)))
      return LOG_IFERR(MDBX_ENODATA);
  }

  if (end) {
    rc = cursor_check_ro(end);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    if (unlikely(!is_pointed(end)))
      return LOG_IFERR(MDBX_ENODATA);
  }

  if (unlikely(begin && end && begin->txn != end->txn && begin->tree != end->tree))
    return LOG_IFERR(MDBX_EINVAL);

  const MDBX_cursor *ref = begin ? begin : end;
  MDBX_cursor *iter = nullptr;
  for (intptr_t i = 0; i < array_size; ++i) {
    if (array[i] != begin && array[i] != end) {
      rc = cursor_check_pure(iter = array[i]);
      if (unlikely(rc != MDBX_SUCCESS))
        return LOG_IFERR(rc);
      if (unlikely(iter->txn != ref->txn && iter->tree != ref->tree))
        return LOG_IFERR(MDBX_EINVAL);
    }
  }
  if (unlikely(!iter))
    return LOG_IFERR(MDBX_EINVAL);

  cursor_couple_t couple;
  if (!begin) {
    begin = cursor_clone_complete(end, &couple);
    rc = outer_first((MDBX_cursor *)begin, nullptr, nullptr);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  if (!end) {
    end = cursor_clone_complete(begin, &couple);
    rc = outer_last((MDBX_cursor *)end, nullptr, nullptr);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  rc = cursor_distribute(begin, end, array, array_size, (deepness < CURSOR_STACK_SIZE) ? deepness : CURSOR_STACK_SIZE);
  return LOG_IFERR(rc);
}

int mdbx_cursor_scroll(MDBX_cursor *mc, intptr_t amount, unsigned deepness) {
  int rc = cursor_check_ro(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  if (unlikely(!is_pointed(mc)))
    return LOG_IFERR(MDBX_ENODATA);

  if (amount > 0)
    rc = cursor_scroll_forward(mc, amount, deepness);
  else if (amount < 0)
    rc = cursor_scroll_backward(mc, -amount, deepness);

  return LOG_IFERR(rc);
}

int mdbx_cursor_distance(const MDBX_cursor *begin, const MDBX_cursor *end, intptr_t *distance, unsigned deepness) {
  if (unlikely(!begin && !end))
    return LOG_IFERR(MDBX_EINVAL);
  if (unlikely(!distance))
    return LOG_IFERR(MDBX_EINVAL);

  int rc;
  if (begin) {
    rc = cursor_check_ro(begin);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    if (unlikely(!is_pointed(begin)))
      return LOG_IFERR(MDBX_ENODATA);
  }

  if (end) {
    rc = cursor_check_ro(end);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    if (unlikely(!is_pointed(end)))
      return LOG_IFERR(MDBX_ENODATA);
  }

  if (unlikely(begin && end && begin->txn != end->txn && begin->tree != end->tree))
    return LOG_IFERR(MDBX_EINVAL);

  cursor_couple_t couple_opposite;
  if (!begin) {
    begin = cursor_clone_complete(end, &couple_opposite);
    rc = outer_first((MDBX_cursor *)begin, nullptr, nullptr);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  if (!end) {
    end = cursor_clone_complete(begin, &couple_opposite);
    rc = outer_last((MDBX_cursor *)end, nullptr, nullptr);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  intptr_t cmp = cursor_cmp(begin, end);
  if (cmp == 0) {
    cmp = (end->flags & z_eof_hard) - (begin->flags & z_eof_hard);
    if (unlikely(cmp != 0)) {
      *distance = (cmp > 0) ? 1 : -1;
      return MDBX_SUCCESS;
    }
    if (cmp == 0 && inner_pointed(end)) {
      ASSERT(inner_pointed(begin));
      cmp = cursor_cmp(&begin->subcur->cursor, &end->subcur->cursor);
      if (cmp == 0) {
        *distance = 0;
        return MDBX_SUCCESS;
      }
    }
  }

  const bool negative = cmp > 0;
  if (negative) {
    const MDBX_cursor *swap = begin;
    begin = end;
    end = swap;
  }

  cursor_couple_t couple_iter;
  MDBX_cursor *iter = (MDBX_cursor *)begin;
  if (iter != &couple_opposite.outer)
    iter = cursor_clone_complete(iter, &couple_iter);

  cdr_t tdr = cursor_distance(iter, end, deepness);
  rc = tdr.err;
  if (likely(rc == MDBX_SUCCESS))
    *distance = (!negative) ? (intptr_t)tdr.distance : -(intptr_t)tdr.distance;

  return LOG_IFERR(rc);
}

int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) {
  return LOG_IFERR(dbi_open(txn, name, flags, dbi, nullptr, nullptr));
}

int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func keycmp,
                      MDBX_cmp_func datacmp) {
  return LOG_IFERR(dbi_open(txn, name, flags, dbi, keycmp, datacmp));
}

static int dbi_open_cstr(MDBX_txn *txn, const char *name_cstr, MDBX_db_flags_t flags, MDBX_dbi *dbi,
                         MDBX_cmp_func keycmp, MDBX_cmp_func datacmp) {
  MDBX_val thunk, *name;
  if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META)
    name = (void *)name_cstr;
  else {
    thunk.iov_len = strlen(name_cstr);
    thunk.iov_base = (void *)name_cstr;
    name = &thunk;
  }
  return dbi_open(txn, name, flags, dbi, keycmp, datacmp);
}

int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) {
  return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, nullptr, nullptr));
}

int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func keycmp,
                     MDBX_cmp_func datacmp) {
  return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, keycmp, datacmp));
}

__cold int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) {
  int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = tbl_purge(&cx.outer);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (!del || dbi < CORE_DBS)
    return MDBX_SUCCESS;

  MDBX_env *const env = txn->env;
  MDBX_val name = env->kvs[dbi].name;
  rc = cursor_init(&cx.outer, txn, MAIN_DBI);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = cursor_seek(&cx.outer, &name, nullptr, MDBX_SET).err;
    if (likely(rc == MDBX_SUCCESS)) {
      cx.outer.next = txn->cursors[MAIN_DBI];
      txn->cursors[MAIN_DBI] = &cx.outer;
      rc = cursor_del(&cx.outer, N_TREE);
      txn->cursors[MAIN_DBI] = cx.outer.next;
      if (likely(rc == MDBX_SUCCESS)) {
        cASSERT0(txn, txn->dbi_state[MAIN_DBI] & DBI_DIRTY);
        cASSERT0(txn, txn->flags & MDBX_TXN_DIRTY);
        txn->dbi_state[dbi] = DBI_LINDO | DBI_OLDEN;
        rc = osal_fastmutex_acquire(&env->dbi_lock);
        if (likely(rc == MDBX_SUCCESS))
          return LOG_IFERR(dbi_close_release(env, dbi));
      }
    }
  }

  txn->flags |= MDBX_TXN_ERROR;
  return LOG_IFERR(rc);
}

__cold int mdbx_dbi_rename(MDBX_txn *txn, MDBX_dbi dbi, const char *name_cstr) {
  MDBX_val thunk, *name;
  if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META)
    name = (void *)name_cstr;
  else {
    thunk.iov_len = strlen(name_cstr);
    thunk.iov_base = (void *)name_cstr;
    name = &thunk;
  }
  return mdbx_dbi_rename2(txn, dbi, name);
}

__cold int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *new_name) {
  int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(new_name == MDBX_CHK_MAIN || new_name->iov_base == MDBX_CHK_MAIN || new_name == MDBX_CHK_GC ||
               new_name->iov_base == MDBX_CHK_GC || new_name == MDBX_CHK_META || new_name->iov_base == MDBX_CHK_META))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(dbi < CORE_DBS))
    return LOG_IFERR(MDBX_EINVAL);
  rc = dbi_check(txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = osal_fastmutex_acquire(&txn->env->dbi_lock);
  if (likely(rc == MDBX_SUCCESS)) {
    struct dbi_rename_result pair = dbi_rename_locked(txn, dbi, *new_name);
    if (pair.defer)
      pair.defer->next = nullptr;
    dbi_defer_release(txn->env, pair.defer);
    rc = pair.err;
  }
  return LOG_IFERR(rc);
}

int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(dbi < CORE_DBS))
    return (dbi == MAIN_DBI) ? MDBX_SUCCESS : LOG_IFERR(MDBX_BAD_DBI);

  if (unlikely(dbi >= env->n_dbi))
    return LOG_IFERR(MDBX_BAD_DBI);

  rc = osal_fastmutex_acquire(&env->dbi_lock);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(dbi >= env->n_dbi)) {
    rc = MDBX_BAD_DBI;
  bailout:
    ENSURE(osal_fastmutex_release(&env->dbi_lock) == MDBX_SUCCESS);
    return LOG_IFERR(rc);
  }

  while (env->basal_txn && (env->dbs_flags[dbi] & DB_VALID) && (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0) {
    /* LY: Опасный код, так как env->txn может быть изменено в другом потоке.
     * К сожалению тут нет надежного решения и может быть падение при неверном
     * использовании API (вызове mdbx_dbi_close конкурентно с завершением
     * пишущей транзакции).
     *
     * Для минимизации вероятности падения сначала проверяем dbi-флаги
     * в basal_txn, а уже после в env->txn. Таким образом, падение может быть
     * только при коллизии с завершением вложенной транзакции.
     *
     * Альтернативно можно попробовать выполнять обновление/put строки в
     * MainDB соответствующей таблице закрываемого хендла. Семантически это
     * верный путь, но проблема в текущем API, в котором исторически dbi-хендл
     * живет и закрывается вне транзакции. Причем проблема не только в том,
     * что нет указателя на текущую пишущую транзакцию, а в том что
     * пользователь точно не ожидает что закрытие хендла приведет к
     * скрытой/непрозрачной активности внутри транзакции потенциально
     * выполняемой в другом потоке. Другими словами, проблема может быть
     * только при неверном использовании API и если пользователь это
     * допускает, то точно не будет ожидать скрытых действий внутри
     * транзакции, и поэтому этот путь потенциально более опасен. */
    const MDBX_txn *const hazard = env->txn;
    osal_compiler_barrier();
    if ((dbi_state(env->basal_txn, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) {
      rc = MDBX_DANGLING_DBI;
      goto bailout;
    }
    osal_memory_barrier();
    if (unlikely(hazard != env->txn))
      continue;
    if (hazard != env->basal_txn && hazard && (hazard->flags & MDBX_TXN_FINISHED) == 0 &&
        hazard->signature == txn_signature &&
        (dbi_state(hazard, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) {
      rc = MDBX_DANGLING_DBI;
      goto bailout;
    }
    osal_compiler_barrier();
    if (likely(hazard == env->txn))
      break;
  }
  rc = dbi_close_release(env, dbi);
  return LOG_IFERR(rc);
}

int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, unsigned *state) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR - MDBX_TXN_PARKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = dbi_check(txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!flags || !state))
    return LOG_IFERR(MDBX_EINVAL);

  *flags = txn->dbs[dbi].flags & DB_PERSISTENT_FLAGS;
  *state = txn->dbi_state[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE);
  return MDBX_SUCCESS;
}

static void stat_get(const tree_t *db, MDBX_stat *st, size_t bytes) {
  st->ms_depth = db->height;
  st->ms_branch_pages = db->branch_pages;
  st->ms_leaf_pages = db->leaf_pages;
  st->ms_overflow_pages = db->large_pages;
  st->ms_entries = db->items;
  if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
    st->ms_mod_txnid = db->mod_txnid;
}

__cold int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, size_t bytes) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = dbi_check(txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(txn->flags & MDBX_TXN_BLOCKED))
    return LOG_IFERR(MDBX_BAD_TXN);

  if (unlikely(txn->dbi_state[dbi] & DBI_STALE)) {
    rc = tbl_refresh((MDBX_txn *)txn, dbi);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  if (unlikely(!dest))
    return LOG_IFERR(MDBX_EINVAL);

  const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid);
  if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid)
    return LOG_IFERR(MDBX_EINVAL);

  stat_get(&txn->dbs[dbi], dest, bytes);
  dest->ms_psize = txn->env->ps;
  return MDBX_SUCCESS;
}

__cold int mdbx_enumerate_tables(const MDBX_txn *txn, MDBX_table_enum_func func, void *ctx) {
  if (unlikely(!func))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, MAIN_DBI);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cx.outer.next = txn->cursors[MAIN_DBI];
  txn->cursors[MAIN_DBI] = &cx.outer;
  for (rc = outer_first(&cx.outer, nullptr, nullptr); rc == MDBX_SUCCESS;
       rc = outer_next(&cx.outer, nullptr, nullptr, MDBX_NEXT_NODUP)) {
    node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
    if (node_flags(node) != N_TREE)
      continue;
    if (unlikely(node_ds(node) != sizeof(tree_t))) {
      ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size",
            (unsigned)node_ds(node));
      rc = MDBX_CORRUPTED;
      break;
    }

    tree_t reside;
    const tree_t *tree = memcpy(&reside, node_data(node), sizeof(reside));
    const MDBX_val name = {node_key(node), node_ks(node)};
    const MDBX_env *const env = txn->env;
    MDBX_dbi dbi = 0;
    for (size_t i = CORE_DBS; i < env->n_dbi; ++i) {
      if (i >= txn->n_dbi || !(env->dbs_flags[i] & DB_VALID))
        continue;
      if (env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[i].name))
        continue;

      tree = dbi_dig(txn, i, &reside);
      dbi = (MDBX_dbi)i;
      break;
    }

    MDBX_stat stat;
    stat_get(tree, &stat, sizeof(stat));
    stat.ms_psize = txn->env->ps;
    rc = func(ctx, txn, &name, tree->flags, &stat, dbi);
    if (rc != MDBX_SUCCESS)
      goto bailout;
  }
  rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;

bailout:
  txn->cursors[MAIN_DBI] = cx.outer.next;
  return LOG_IFERR(rc);
}

__cold static intptr_t reasonable_db_maxsize(void) {
  static intptr_t cached_result;
  if (cached_result == 0) {
    intptr_t pagesize, total_ram_pages;
    if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) != MDBX_SUCCESS))
      /* the 32-bit limit is good enough for fallback */
      return cached_result = MAX_MAPSIZE32 / 4;

#if defined(__SANITIZE_ADDRESS__)
    total_ram_pages >>= 4;
#endif /* __SANITIZE_ADDRESS__ */
    if (RUNNING_ON_VALGRIND)
      total_ram_pages >>= 4;

    if (unlikely((size_t)total_ram_pages > MAX_MAPSIZE / (size_t)pagesize))
      return cached_result = MAX_MAPSIZE / 2;
    ASSERT(MAX_MAPSIZE >= (size_t)(total_ram_pages * pagesize * 2));

    /* Suggesting should not be more than golden ratio of the size of RAM. */
    cached_result = (intptr_t)((size_t)total_ram_pages * 207 >> 7) * pagesize;

    /* Round to the nearest human-readable granulation. */
    for (size_t unit = MEGABYTE; unit; unit <<= 5) {
      const size_t floor = floor_powerof2(cached_result, unit);
      const size_t ceil = ceil_powerof2(cached_result, unit);
      const size_t threshold = (size_t)cached_result >> 4;
      const bool down = cached_result - floor < ceil - cached_result || ceil > MAX_MAPSIZE / 4;
      if (threshold < (down ? cached_result - floor : ceil - cached_result))
        break;
      cached_result = down ? floor : ceil;
    }
  }
  return cached_result;
}

__cold static int check_alternative_lck_absent(const pathchar_t *lck_pathname) {
  int err = osal_fileexists(lck_pathname);
  if (unlikely(err != MDBX_RESULT_FALSE)) {
    if (err == MDBX_RESULT_TRUE)
      err = MDBX_DUPLICATED_CLK;
    ERROR("Alternative/Duplicate LCK-file '%" MDBX_PRIsPATH "' error %d", lck_pathname, err);
  }
  return err;
}

__cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, const mdbx_mode_t mode) {
  memset(&env->pathname, 0, sizeof(env->pathname));
  if (unlikely(!pathname || !*pathname))
    return MDBX_EINVAL;

  int rc;
#if defined(_WIN32) || defined(_WIN64)
  const DWORD dwAttrib = GetFileAttributesW(pathname);
  if (dwAttrib == INVALID_FILE_ATTRIBUTES) {
    rc = GetLastError();
    if (rc != MDBX_ENOFILE)
      return rc;
    if (mode == 0 || (env->flags & MDBX_RDONLY) != 0)
      /* can't open existing */
      return rc;

    /* auto-create directory if requested */
    if ((env->flags & MDBX_NOSUBDIR) == 0 && !CreateDirectoryW(pathname, nullptr)) {
      rc = GetLastError();
      if (rc != ERROR_ALREADY_EXISTS)
        return rc;
    }
  } else {
    /* ignore passed MDBX_NOSUBDIR flag and set it automatically */
    env->flags |= MDBX_NOSUBDIR;
    if (dwAttrib & FILE_ATTRIBUTE_DIRECTORY)
      env->flags -= MDBX_NOSUBDIR;
  }
#else
  struct stat st;
  if (stat(pathname, &st) != 0) {
    rc = errno;
    if (rc != MDBX_ENOFILE)
      return rc;
    if (mode == 0 || (env->flags & MDBX_RDONLY) != 0)
      /* can't open non-existing */
      return rc /* MDBX_ENOFILE */;

    /* auto-create directory if requested */
    const mdbx_mode_t dir_mode =
        (/* inherit read/write permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
        /* always add read/write/search for owner */ S_IRWXU |
        ((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) |
        ((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0);
    if ((env->flags & MDBX_NOSUBDIR) == 0 && mkdir(pathname, dir_mode)) {
      rc = errno;
      if (rc != EEXIST)
        return rc;
    }
  } else {
    /* ignore passed MDBX_NOSUBDIR flag and set it automatically */
    env->flags |= MDBX_NOSUBDIR;
    if (S_ISDIR(st.st_mode))
      env->flags -= MDBX_NOSUBDIR;
  }
#endif

  static const pathchar_t dxb_name[] = MDBX_DATANAME;
  static const pathchar_t lck_name[] = MDBX_LOCKNAME;
  static const pathchar_t lock_suffix[] = MDBX_LOCK_SUFFIX;

#if defined(_WIN32) || defined(_WIN64)
  ASSERT(dxb_name[0] == '\\' && lck_name[0] == '\\');
  const size_t pathname_len = wcslen(pathname);
#else
  ASSERT(dxb_name[0] == '/' && lck_name[0] == '/');
  const size_t pathname_len = strlen(pathname);
#endif
  ASSERT(!osal_isdirsep(lock_suffix[0]));
  size_t base_len = pathname_len;
  static const size_t dxb_name_len = ARRAY_LENGTH(dxb_name) - 1;
  if (env->flags & MDBX_NOSUBDIR) {
    if (base_len > dxb_name_len && osal_pathequal(pathname + base_len - dxb_name_len, dxb_name, dxb_name_len)) {
      env->flags -= MDBX_NOSUBDIR;
      base_len -= dxb_name_len;
    } else if (base_len == dxb_name_len - 1 && osal_isdirsep(dxb_name[0]) && osal_isdirsep(lck_name[0]) &&
               osal_pathequal(pathname + base_len - dxb_name_len + 1, dxb_name + 1, dxb_name_len - 1)) {
      env->flags -= MDBX_NOSUBDIR;
      base_len -= dxb_name_len - 1;
    }
  }

  const size_t suflen_with_NOSUBDIR = sizeof(lock_suffix) + sizeof(pathchar_t);
  const size_t suflen_without_NOSUBDIR = sizeof(lck_name) + sizeof(dxb_name);
  const size_t enough4any =
      (suflen_with_NOSUBDIR > suflen_without_NOSUBDIR) ? suflen_with_NOSUBDIR : suflen_without_NOSUBDIR;
  const size_t bytes_needed = sizeof(pathchar_t) * (base_len * 2 + pathname_len + 1) + enough4any;
  env->pathname.buffer = osal_malloc(bytes_needed);
  if (!env->pathname.buffer)
    return MDBX_ENOMEM;

  env->pathname.specified = env->pathname.buffer;
  env->pathname.dxb = env->pathname.specified + pathname_len + 1;
  env->pathname.lck = env->pathname.dxb + base_len + dxb_name_len + 1;
  rc = MDBX_SUCCESS;
  pathchar_t *const buf = env->pathname.buffer;
  if (base_len) {
    memcpy(buf, pathname, sizeof(pathchar_t) * pathname_len);
    if (env->flags & MDBX_NOSUBDIR) {
      const pathchar_t *const lck_ext = osal_fileext(lck_name, ARRAY_LENGTH(lck_name));
      if (lck_ext) {
        pathchar_t *pathname_ext = osal_fileext(buf, pathname_len);
        memcpy(pathname_ext ? pathname_ext : buf + pathname_len, lck_ext,
               sizeof(pathchar_t) * (ARRAY_END(lck_name) - lck_ext));
        rc = check_alternative_lck_absent(buf);
      }
    } else {
      memcpy(buf + base_len, dxb_name, sizeof(dxb_name));
      memcpy(buf + base_len + dxb_name_len, lock_suffix, sizeof(lock_suffix));
      rc = check_alternative_lck_absent(buf);
    }

    memcpy(env->pathname.dxb, pathname, sizeof(pathchar_t) * (base_len + 1));
    memcpy(env->pathname.lck, pathname, sizeof(pathchar_t) * base_len);
    if (env->flags & MDBX_NOSUBDIR) {
      memcpy(env->pathname.lck + base_len, lock_suffix, sizeof(lock_suffix));
    } else {
      memcpy(env->pathname.dxb + base_len, dxb_name, sizeof(dxb_name));
      memcpy(env->pathname.lck + base_len, lck_name, sizeof(lck_name));
    }
  } else {
    ASSERT(!(env->flags & MDBX_NOSUBDIR));
    memcpy(buf, dxb_name + 1, sizeof(dxb_name) - sizeof(pathchar_t));
    memcpy(buf + dxb_name_len - 1, lock_suffix, sizeof(lock_suffix));
    rc = check_alternative_lck_absent(buf);

    memcpy(env->pathname.dxb, dxb_name + 1, sizeof(dxb_name) - sizeof(pathchar_t));
    memcpy(env->pathname.lck, lck_name + 1, sizeof(lck_name) - sizeof(pathchar_t));
  }

  memcpy(env->pathname.specified, pathname, sizeof(pathchar_t) * (pathname_len + 1));
  return rc;
}

/*----------------------------------------------------------------------------*/

__cold int mdbx_env_create(MDBX_env **penv) {
  if (unlikely(!penv))
    return LOG_IFERR(MDBX_EINVAL);
  *penv = nullptr;

#ifdef MDBX_HAVE_C11ATOMICS
  if (unlikely(!atomic_is_lock_free((const volatile uint32_t *)penv))) {
    ERROR("lock-free atomic ops for %u-bit types is required", 32);
    return LOG_IFERR(MDBX_INCOMPATIBLE);
  }
#if MDBX_64BIT_ATOMIC
  if (unlikely(!atomic_is_lock_free((const volatile uint64_t *)penv))) {
    ERROR("lock-free atomic ops for %u-bit types is required", 64);
    return LOG_IFERR(MDBX_INCOMPATIBLE);
  }
#endif /* MDBX_64BIT_ATOMIC */
#endif /* MDBX_HAVE_C11ATOMICS */

  if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) {
    ERROR("unsuitable system pagesize %u", globals.sys_pagesize);
    return LOG_IFERR(MDBX_INCOMPATIBLE);
  }

#if defined(__linux__) || defined(__gnu_linux__)
  if (unlikely(globals.linux_kernel_version < 0x03100000)) {
    /* 2025-08-05: Ядро 3.16 выпущено 11 лет назад и было самым долго поддерживаемым из 3.x до июля 2020.
     * Три года назад (в 2022) здесь была заблокирована работа на ядрах меньше 4.x, как устаревших и для которых
     * крайне затруднительно обеспечить какое-либо тестирование. Теперь же я решил изменить решение и разрешить
     * работу на старых ядрах начиная с 3.16, логика тут такая:
     *  - поведение старых ядер уже точно не будет меняться,
     *    а в текущем коде libmdbx есть всё необходимое для работы начиная с 3.16;
     *  - есть широко-используемые проекты (Isar), которым требуется поддержка старых ядер;
     *  - сейчас тестирование для 4.x также затруднено, как и для 3.16, уже не приносит какого-либо облегчения
     *    с тестированием и мне приходится полагаться на гарантии совместимости API ядра и glibc/musl;
     *  - использование возможностей из новых ядер всё равно требует проверок/ветвлений;
     *  = поэтому сейчас нет причин отказываться от работы на 3.16 поддерживая ядра 4.0 */
    ERROR("too old linux kernel %u.%u.%u.%u, the >= 3.16 is required", globals.linux_kernel_version >> 24,
          (globals.linux_kernel_version >> 16) & 255, (globals.linux_kernel_version >> 8) & 255,
          globals.linux_kernel_version & 255);
    return LOG_IFERR(MDBX_INCOMPATIBLE);
  }
#endif /* Linux */

  MDBX_env *env = osal_calloc(1, sizeof(MDBX_env));
  if (unlikely(!env))
    return LOG_IFERR(MDBX_ENOMEM);

  env->max_readers = DEFAULT_READERS;
  env->max_dbi = env->n_dbi = CORE_DBS;
  env->lazy_fd = env->dsync_fd = env->fd4meta = env->lck_mmap.fd = INVALID_HANDLE_VALUE;
  env->stuck_meta = -1;

  env_options_init(env);
  env_setup_pagesize(env, (globals.sys_pagesize < MDBX_MAX_PAGESIZE) ? globals.sys_pagesize : MDBX_MAX_PAGESIZE);

  int rc = osal_fastmutex_init(&env->dbi_lock);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

#if defined(_WIN32) || defined(_WIN64)
  imports.srwl_Init(&env->remap_lock);
  InitializeCriticalSection(&env->lck_event_cs);
  InitializeCriticalSection(&env->dxb_event_cs);
#else
  rc = osal_fastmutex_init(&env->remap_lock);
  if (unlikely(rc != MDBX_SUCCESS)) {
    osal_fastmutex_destroy(&env->dbi_lock);
    goto bailout;
  }

#if MDBX_LOCKING > MDBX_LOCKING_SYSV
  lck_t *const stub = lckless_stub(env);
  rc = lck_ipclock_stubinit(&stub->wrt_lock);
#endif /* MDBX_LOCKING */
  if (unlikely(rc != MDBX_SUCCESS)) {
    osal_fastmutex_destroy(&env->remap_lock);
    osal_fastmutex_destroy(&env->dbi_lock);
    goto bailout;
  }
#endif /* Windows */

  VALGRIND_CREATE_MEMPOOL(env, 0, 0);
  env->signature.weak = env_signature;
  *penv = env;
  return MDBX_SUCCESS;

bailout:
  osal_free(env);
  return LOG_IFERR(rc);
}

__cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) {
  if (unlikely(target >= NUM_METAS))
    return LOG_IFERR(MDBX_EINVAL);
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely((env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_EXCLUSIVE))
    return LOG_IFERR(MDBX_EPERM);

  const meta_t *const target_meta = METAPAGE(env, target);
  txnid_t new_txnid = constmeta_txnid(target_meta);
  if (new_txnid < MIN_TXNID)
    new_txnid = MIN_TXNID;
  for (unsigned n = 0; n < NUM_METAS; ++n) {
    if (n == target)
      continue;
    page_t *const page = pgno2page(env, n);
    meta_t meta = *page_meta(page);
    if (meta_validate(env, &meta, page, n, nullptr) != MDBX_SUCCESS) {
      int err = meta_override(env, n, 0, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return LOG_IFERR(err);
    } else {
      txnid_t txnid = constmeta_txnid(&meta);
      if (new_txnid <= txnid)
        new_txnid = safe64_txnid_next(txnid);
    }
  }

  if (unlikely(new_txnid > MAX_TXNID)) {
    ERROR("txnid overflow, raise %d", MDBX_TXN_FULL);
    return LOG_IFERR(MDBX_TXN_FULL);
  }
  return LOG_IFERR(meta_override(env, target, new_txnid, target_meta));
}

__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, unsigned target_meta, bool writeable) {
#if defined(_WIN32) || defined(_WIN64)
  wchar_t *pathnameW = nullptr;
  int rc = osal_mb2w(pathname, &pathnameW);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = mdbx_env_open_for_recoveryW(env, pathnameW, target_meta, writeable);
    osal_free(pathnameW);
  }
  return LOG_IFERR(rc);
}

__cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, unsigned target_meta, bool writeable) {
#endif /* Windows */

  if (unlikely(target_meta >= NUM_METAS))
    return LOG_IFERR(MDBX_EINVAL);
  int rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  if (unlikely(env->dxb_mmap.base))
    return LOG_IFERR(MDBX_EPERM);

  env->stuck_meta = (int8_t)target_meta;
  return
#if defined(_WIN32) || defined(_WIN64)
      mdbx_env_openW
#else
      mdbx_env_open
#endif /* Windows */
      (env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY, 0);
}

__cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) {
#if defined(_WIN32) || defined(_WIN64)
  wchar_t *pathnameW = nullptr;
  int rc = osal_mb2w(pathname, &pathnameW);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = mdbx_env_deleteW(pathnameW, mode);
    osal_free(pathnameW);
  }
  return LOG_IFERR(rc);
}

__cold int mdbx_env_deleteW(const wchar_t *pathname, MDBX_env_delete_mode_t mode) {
#endif /* Windows */

  switch (mode) {
  default:
    return LOG_IFERR(MDBX_EINVAL);
  case MDBX_ENV_JUST_DELETE:
  case MDBX_ENV_ENSURE_UNUSED:
  case MDBX_ENV_WAIT_FOR_UNUSED:
    break;
  }

#ifdef __e2k__ /* https://bugs.mcst.ru/bugzilla/show_bug.cgi?id=6011 */
  MDBX_env *const dummy_env = alloca(sizeof(MDBX_env));
#else
  MDBX_env dummy_env_silo, *const dummy_env = &dummy_env_silo;
#endif
  memset(dummy_env, 0, sizeof(*dummy_env));
  dummy_env->flags = (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS;
  dummy_env->ps = (unsigned)mdbx_default_pagesize();

  STATIC_ASSERT(sizeof(dummy_env->flags) == sizeof(MDBX_env_flags_t));
  int rc = MDBX_RESULT_TRUE, err = env_handle_pathname(dummy_env, pathname, 0);
  if (likely(err == MDBX_SUCCESS)) {
    mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE, dxb_handle = INVALID_HANDLE_VALUE;
    if (mode > MDBX_ENV_JUST_DELETE) {
      err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.dxb, &dxb_handle, 0);
      err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
      if (err == MDBX_SUCCESS) {
        err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.lck, &clk_handle, 0);
        err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
      }
      if (err == MDBX_SUCCESS && clk_handle != INVALID_HANDLE_VALUE)
        err = osal_lockfile(clk_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED);
      if (err == MDBX_SUCCESS && dxb_handle != INVALID_HANDLE_VALUE)
        err = osal_lockfile(dxb_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED);
    }

    if (err == MDBX_SUCCESS) {
      err = osal_removefile(dummy_env->pathname.dxb);
      if (err == MDBX_SUCCESS)
        rc = MDBX_SUCCESS;
      else if (err == MDBX_ENOFILE)
        err = MDBX_SUCCESS;
    }

    if (err == MDBX_SUCCESS) {
      err = osal_removefile(dummy_env->pathname.lck);
      if (err == MDBX_SUCCESS)
        rc = MDBX_SUCCESS;
      else if (err == MDBX_ENOFILE)
        err = MDBX_SUCCESS;
    }

    if (err == MDBX_SUCCESS && !(dummy_env->flags & MDBX_NOSUBDIR) &&
        (/* pathname != "." */ pathname[0] != '.' || pathname[1] != 0) &&
        (/* pathname != ".." */ pathname[0] != '.' || pathname[1] != '.' || pathname[2] != 0)) {
      err = osal_removedirectory(pathname);
      if (err == MDBX_SUCCESS)
        rc = MDBX_SUCCESS;
      else if (err == MDBX_ENOFILE)
        err = MDBX_SUCCESS;
    }

    if (dxb_handle != INVALID_HANDLE_VALUE)
      osal_closefile(dxb_handle);
    if (clk_handle != INVALID_HANDLE_VALUE)
      osal_closefile(clk_handle);
  } else if (err == MDBX_ENOFILE)
    err = MDBX_SUCCESS;

  osal_free(dummy_env->pathname.buffer);
  return LOG_IFERR((err == MDBX_SUCCESS) ? rc : err);
}

__cold int mdbx_env_open(MDBX_env *env, const char *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) {
#if defined(_WIN32) || defined(_WIN64)
  wchar_t *pathnameW = nullptr;
  int rc = osal_mb2w(pathname, &pathnameW);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = mdbx_env_openW(env, pathnameW, flags, mode);
    osal_free(pathnameW);
    if (rc == MDBX_SUCCESS)
      /* force to make cache of the multi-byte pathname representation */
      mdbx_env_get_path(env, &pathname);
  }
  return LOG_IFERR(rc);
}

__cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) {
#endif /* Windows */

  int rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(flags & ~ENV_USABLE_FLAGS))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(env->lazy_fd != INVALID_HANDLE_VALUE || (env->flags & ENV_ACTIVE) != 0 || env->dxb_mmap.base))
    return LOG_IFERR(MDBX_EPERM);

  /* Pickup previously mdbx_env_set_flags(),
   * but avoid MDBX_UTTERLY_NOSYNC by disjunction */
  const uint32_t saved_me_flags = env->flags;
  flags = combine_durability_flags(flags | DEPRECATED_COALESCE, env->flags);

  if (flags & MDBX_RDONLY) {
    /* Silently ignore irrelevant flags when we're only getting read access */
    flags &= ~(MDBX_WRITEMAP | DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_COALESCE |
               MDBX_LIFORECLAIM | MDBX_NOMEMINIT | MDBX_ACCEDE);
    mode = 0;
  } else {
#if MDBX_MMAP_INCOHERENT_FILE_WRITE
    /* Temporary `workaround` for OpenBSD kernel's flaw.
     * See https://libmdbx.dqdkfa.ru/dead-github/issues/67 */
    if ((flags & MDBX_WRITEMAP) == 0) {
      if (flags & MDBX_ACCEDE)
        flags |= MDBX_WRITEMAP;
      else {
        debug_log(MDBX_LOG_ERROR, __func__, __LINE__,
                  "System (i.e. OpenBSD) requires MDBX_WRITEMAP because "
                  "of an internal flaw(s) in a file/buffer/page cache.\n");
        return LOG_IFERR(42 /* ENOPROTOOPT */);
      }
    }
#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
  }

  env->flags = (flags & ~ENV_FATAL_ERROR);
  rc = env_handle_pathname(env, pathname, mode);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

  env->kvs = osal_calloc(env->max_dbi, sizeof(env->kvs[0]));
  env->dbs_flags = osal_calloc(env->max_dbi, sizeof(env->dbs_flags[0]));
  env->dbi_seqs = osal_calloc(env->max_dbi, sizeof(env->dbi_seqs[0]));
  if (unlikely(!(env->kvs && env->dbs_flags && env->dbi_seqs))) {
    rc = MDBX_ENOMEM;
    goto bailout;
  }

  if ((flags & MDBX_RDONLY) == 0) {
    env->basal_txn = txn_basal_create(env->max_dbi);
    if (unlikely(!env->basal_txn)) {
      rc = MDBX_ENOMEM;
      goto bailout;
    }
    env->basal_txn->env = env;
    env_options_adjust_defaults(env);
  }

  rc = env_open(env, mode);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

#if MDBX_DEBUG > 0
  const troika_t troika = meta_tap(env);
  const meta_ptr_t head = meta_recent(env, &troika);
  const tree_t *db = &head.ptr_c->trees.main;

  DEBUG("opened database version %u, pagesize %u", (uint8_t)unaligned_peek_u64(4, head.ptr_c->magic_and_version),
        env->ps);
  DEBUG("using meta page %" PRIaPGNO ", txn %" PRIaTXN, payload2page(head.ptr_c)->pgno, head.txnid);
  DEBUG("depth: %u", db->height);
  DEBUG("entries: %" PRIu64, db->items);
  DEBUG("branch pages: %" PRIaPGNO, db->branch_pages);
  DEBUG("leaf pages: %" PRIaPGNO, db->leaf_pages);
  DEBUG("large/overflow pages: %" PRIaPGNO, db->large_pages);
  DEBUG("root: %" PRIaPGNO, db->root);
  DEBUG("schema_altered: %" PRIaTXN, db->mod_txnid);
#endif /* MDBX_DEBUG > 0 */

  if (likely(rc == MDBX_SUCCESS)) {
    dxb_sanitize_tail(env, nullptr);
  } else {
  bailout:
    if (likely(env_close(env, false) == MDBX_SUCCESS)) {
      env->flags = saved_me_flags;
    } else {
      rc = MDBX_PANIC;
      env->flags = saved_me_flags | ENV_FATAL_ERROR;
    }
  }
  return LOG_IFERR(rc);
}

/*----------------------------------------------------------------------------*/

#if !(defined(_WIN32) || defined(_WIN64))
__cold int mdbx_env_resurrect_after_fork(MDBX_env *env) {
  if (unlikely(!env))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(env->signature.weak != env_signature))
    return LOG_IFERR(MDBX_EBADSIGN);

  if (unlikely(env->flags & ENV_FATAL_ERROR))
    return LOG_IFERR(MDBX_PANIC);

  if (unlikely((env->flags & ENV_ACTIVE) == 0))
    return MDBX_SUCCESS;

  const mdbx_pid_t new_pid = osal_getpid();
  if (unlikely(env->pid == new_pid))
    return MDBX_SUCCESS;

  if (!atomic_cas32(&env->signature, env_signature, ~env_signature))
    return LOG_IFERR(MDBX_EBADSIGN);

  if (env->txn)
    txn_abort_after_resurrect(env->basal_txn);
  env->registered_reader_pid = 0;
  int rc = env_close(env, true);
  env->signature.weak = env_signature;
  if (likely(rc == MDBX_SUCCESS)) {
    rc = (env->flags & MDBX_EXCLUSIVE) ? MDBX_BUSY : env_open(env, 0);
    if (unlikely(rc != MDBX_SUCCESS && env_close(env, false) != MDBX_SUCCESS)) {
      rc = MDBX_PANIC;
      env->flags |= ENV_FATAL_ERROR;
    }
  }
  return LOG_IFERR(rc);
}
#endif /* Windows */

__cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) {
  page_t *dp;
  int rc = MDBX_SUCCESS;

  if (unlikely(!env))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(env->signature.weak != env_signature))
    return LOG_IFERR(MDBX_EBADSIGN);

#if MDBX_ENV_CHECKPID || !(defined(_WIN32) || defined(_WIN64))
  /* Check the PID even if MDBX_ENV_CHECKPID=0 on non-Windows
   * platforms (i.e. where fork() is available).
   * This is required to legitimize a call after fork()
   * from a child process, that should be allowed to free resources. */
  if (unlikely(env->pid != osal_getpid()))
    env->flags |= ENV_FATAL_ERROR;
#endif /* MDBX_ENV_CHECKPID */

  if (env->dxb_mmap.base && (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0 && env->basal_txn) {
    if (env->basal_txn->owner && env->basal_txn->owner != osal_thread_self())
      return LOG_IFERR(MDBX_BUSY);
  } else
    dont_sync = true;

  if (!atomic_cas32(&env->signature, env_signature, 0))
    return LOG_IFERR(MDBX_EBADSIGN);

  if (!dont_sync) {
#if defined(_WIN32) || defined(_WIN64)
    /* On windows, without blocking is impossible to determine whether another
     * process is running a writing transaction or not.
     * Because in the "owner died" condition kernel don't release
     * file lock immediately. */
    rc = env_sync(env, true, false);
    rc = (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
#else
    struct stat st;
    if (unlikely(fstat(env->lazy_fd, &st)))
      rc = errno;
    else if (st.st_nlink > 0 /* don't sync deleted files */) {
      rc = env_sync(env, true, true);
      rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
            rc == MDBX_RESULT_TRUE)
               ? MDBX_SUCCESS
               : rc;
    }
#endif /* Windows */
  }

  if (env->basal_txn && (MDBX_TXN_CHECKOWNER ? env->basal_txn->owner == osal_thread_self() : !!env->basal_txn->owner))
    lck_txn_unlock(env);

  eASSERT0(env, env->signature.weak == 0);
  rc = env_close(env, false) ? MDBX_PANIC : rc;
  ENSURE_OBJ(env, osal_fastmutex_destroy(&env->dbi_lock) == MDBX_SUCCESS);
#if defined(_WIN32) || defined(_WIN64)
  /* remap_lock don't have destructor (Slim Reader/Writer Lock) */
  DeleteCriticalSection(&env->lck_event_cs);
  DeleteCriticalSection(&env->dxb_event_cs);
#else
  ENSURE_OBJ(env, osal_fastmutex_destroy(&env->remap_lock) == MDBX_SUCCESS);
#endif /* Windows */

#if MDBX_LOCKING > MDBX_LOCKING_SYSV
  lck_t *const stub = lckless_stub(env);
  /* может вернуть ошибку в дочернем процессе после fork() */
  lck_ipclock_destroy(&stub->wrt_lock);
#endif /* MDBX_LOCKING */

  while ((dp = env->shadow_reserve) != nullptr) {
    MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->ps);
    VALGRIND_MAKE_MEM_DEFINED(&page_next(dp), sizeof(page_t *));
    env->shadow_reserve = page_next(dp);
    void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t));
    osal_free(ptr);
  }
  VALGRIND_DESTROY_MEMPOOL(env);
  osal_free(env);

  return LOG_IFERR(rc);
}

/*----------------------------------------------------------------------------*/

__must_check_result static int env_info_sys(const MDBX_env *env, MDBX_envinfo *out) {
  out->mi_bootid.current.x = globals.bootid.x;
  out->mi_bootid.current.y = globals.bootid.y;
  out->mi_sys_pagesize = globals.sys_pagesize;
#ifdef __OpenBSD__
  out->mi_sys_upcblk = 0;
#elif defined(_WIN32) || defined(_WIN64)
  out->mi_sys_upcblk = globals.sys_allocation_granularity;
#elif defined(AT_UCACHEBSIZE)
  out->mi_sys_upcblk = globals.sys_unified_cache_block;
#else
  out->mi_sys_upcblk = globals.sys_pagesize;
#endif /* AT_UCACHEBSIZE */

  out->mi_dxb_fsize = 0;
  out->mi_dxb_fallocated = 0;
  out->mi_sys_ioblk = 0;
  if (env->dxb_mmap.fd != INVALID_HANDLE_VALUE) {
#if defined(_WIN32) || defined(_WIN64)
    union {
      BY_HANDLE_FILE_INFORMATION bh;
      FILE_STANDARD_INFO std;
#if _WIN32_WINNT >= _WIN32_WINNT_WIN8
      FILE_STORAGE_INFO storage;
#endif
    } sys_finfo;
    if (imports.GetFileInformationByHandleEx &&
        imports.GetFileInformationByHandleEx(env->dxb_mmap.fd, FileStandardInfo, &sys_finfo.std,
                                             sizeof(sys_finfo.std))) {
      out->mi_dxb_fsize = sys_finfo.std.EndOfFile.QuadPart;
      out->mi_dxb_fallocated = sys_finfo.std.AllocationSize.QuadPart;
#if _WIN32_WINNT >= _WIN32_WINNT_WIN8
      if (imports.GetFileInformationByHandleEx(env->dxb_mmap.fd, FileStorageInfo, &sys_finfo.storage,
                                               sizeof(sys_finfo.storage))) {
        out->mi_sys_ioblk = (sys_finfo.storage.FileSystemEffectivePhysicalBytesPerSectorForAtomicity >
                             sys_finfo.storage.LogicalBytesPerSector)
                                ? sys_finfo.storage.FileSystemEffectivePhysicalBytesPerSectorForAtomicity
                                : sys_finfo.storage.LogicalBytesPerSector;
      }
#endif
    } else if (GetFileInformationByHandle(env->dxb_mmap.fd, &sys_finfo.bh)) {
      out->mi_dxb_fsize = sys_finfo.bh.nFileSizeLow | (uint64_t)sys_finfo.bh.nFileSizeHigh << 32;
    } else
      return GetLastError();
#else
    struct stat sys_fstat;
    if (fstat(env->dxb_mmap.fd, &sys_fstat))
      return errno;
    out->mi_dxb_fsize = sys_fstat.st_size;
    out->mi_dxb_fallocated = UINT64_C(512) * sys_fstat.st_blocks;
    out->mi_sys_ioblk = sys_fstat.st_blksize;
#endif /* !Windows */
  }
  return MDBX_SUCCESS;
}

__must_check_result static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out,
                                             troika_t *const troika) {
  int err = env_info_sys(env, out);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  if (unlikely(env->flags & ENV_FATAL_ERROR))
    return MDBX_PANIC;

  /* is the environment open?
   * (https://libmdbx.dqdkfa.ru/dead-github/issues/171) */
  if (unlikely(!env->dxb_mmap.base)) {
    /* environment not yet opened */
#if 1
    /* default behavior: returns the available info but zeroed the rest */
    out->mi_geo.lower = env->geo_in_bytes.lower;
    out->mi_geo.upper = env->geo_in_bytes.upper;
    out->mi_geo.shrink = env->geo_in_bytes.shrink;
    out->mi_geo.grow = env->geo_in_bytes.grow;
    out->mi_geo.current = env->geo_in_bytes.now;
    out->mi_maxreaders = env->max_readers;
    out->mi_dxb_pagesize = env->ps;
    return MDBX_SUCCESS;
#else
    /* some users may prefer this behavior: return appropriate error */
    return MDBX_EPERM;
#endif
  }

  *troika = (txn && !(txn->flags & txn_ro_flat)) ? txn->wr.troika : meta_tap(env);
  const meta_ptr_t head = meta_recent(env, troika);
  const meta_t *const meta0 = METAPAGE(env, 0);
  const meta_t *const meta1 = METAPAGE(env, 1);
  const meta_t *const meta2 = METAPAGE(env, 2);
  out->mi_recent_txnid = head.txnid;
  out->mi_meta_txnid[0] = troika->txnid[0];
  out->mi_meta_sign[0] = unaligned_peek_u64(4, meta0->sign);
  out->mi_meta_txnid[1] = troika->txnid[1];
  out->mi_meta_sign[1] = unaligned_peek_u64(4, meta1->sign);
  out->mi_meta_txnid[2] = troika->txnid[2];
  out->mi_meta_sign[2] = unaligned_peek_u64(4, meta2->sign);
  memcpy(&out->mi_bootid.meta[0], &meta0->bootid, 16);
  memcpy(&out->mi_bootid.meta[1], &meta1->bootid, 16);
  memcpy(&out->mi_bootid.meta[2], &meta2->bootid, 16);
  memcpy(&out->mi_dxbid, &meta0->dxbid, 16);

  const volatile meta_t *txn_meta = head.ptr_v;
  out->mi_last_pgno = txn_meta->geometry.first_unallocated - 1;
  out->mi_geo.current = pgno2bytes(env, txn_meta->geometry.now);
  if (txn) {
    out->mi_last_pgno = txn->geo.first_unallocated - 1;
    out->mi_geo.current = pgno2bytes(env, txn->geo.end_pgno);

    const txnid_t wanna_meta_txnid = txn_basis_snapshot(txn);
    txn_meta = (out->mi_meta_txnid[0] == wanna_meta_txnid) ? meta0 : txn_meta;
    txn_meta = (out->mi_meta_txnid[1] == wanna_meta_txnid) ? meta1 : txn_meta;
    txn_meta = (out->mi_meta_txnid[2] == wanna_meta_txnid) ? meta2 : txn_meta;
  }
  out->mi_geo.lower = pgno2bytes(env, txn_meta->geometry.lower);
  out->mi_geo.upper = pgno2bytes(env, txn_meta->geometry.upper);
  out->mi_geo.shrink = pgno2bytes(env, pv2pages(txn_meta->geometry.shrink_pv));
  out->mi_geo.grow = pgno2bytes(env, pv2pages(txn_meta->geometry.grow_pv));
  out->mi_mapsize = env->dxb_mmap.limit;

  const lck_t *const lck = env->lck;
  out->mi_maxreaders = env->max_readers;
  out->mi_numreaders = env->lck_mmap.lck ? atomic_load32(&lck->rdt_length, mo_Relaxed) : INT32_MAX;
  out->mi_dxb_pagesize = env->ps;

  const uint64_t unsynced_pages = atomic_load64(&lck->unsynced_pages, mo_Relaxed) +
                                  ((uint32_t)out->mi_recent_txnid != atomic_load32(&lck->meta_sync_txnid, mo_Relaxed));
  out->mi_unsync_volume = pgno2bytes(env, (size_t)unsynced_pages);
  const uint64_t monotime_now = osal_monotime();
  uint64_t ts = atomic_load64(&lck->eoos_timestamp, mo_Relaxed);
  out->mi_since_sync_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0;
  ts = atomic_load64(&lck->readers_check_timestamp, mo_Relaxed);
  out->mi_since_reader_check_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0;
  out->mi_autosync_threshold = pgno2bytes(env, atomic_load32(&lck->autosync_threshold, mo_Relaxed));
  out->mi_autosync_period_seconds16dot16 =
      osal_monotime_to_16dot16_noUnderflow(atomic_load64(&lck->autosync_period, mo_Relaxed));
  out->mi_bootid.current.x = globals.bootid.x;
  out->mi_bootid.current.y = globals.bootid.y;
  out->mi_mode = env->lck_mmap.lck ? lck->envmode.weak : env->flags;

  out->mi_pgop_stat.newly = atomic_load64(&lck->pgops.newly, mo_Relaxed);
  out->mi_pgop_stat.cow = atomic_load64(&lck->pgops.cow, mo_Relaxed);
  out->mi_pgop_stat.clone = atomic_load64(&lck->pgops.clone, mo_Relaxed);
  out->mi_pgop_stat.split = atomic_load64(&lck->pgops.split, mo_Relaxed);
  out->mi_pgop_stat.merge = atomic_load64(&lck->pgops.merge, mo_Relaxed);
  out->mi_pgop_stat.spill = atomic_load64(&lck->pgops.spill, mo_Relaxed);
  out->mi_pgop_stat.unspill = atomic_load64(&lck->pgops.unspill, mo_Relaxed);
  out->mi_pgop_stat.wops = atomic_load64(&lck->pgops.wops, mo_Relaxed);
  out->mi_pgop_stat.prefault = atomic_load64(&lck->pgops.prefault, mo_Relaxed);
  out->mi_pgop_stat.mincore = atomic_load64(&lck->pgops.mincore, mo_Relaxed);
  out->mi_pgop_stat.msync = atomic_load64(&lck->pgops.msync, mo_Relaxed);
  out->mi_pgop_stat.fsync = atomic_load64(&lck->pgops.fsync, mo_Relaxed);

  txnid_t overall_latter_reader_txnid = out->mi_recent_txnid;
  txnid_t self_latter_reader_txnid = overall_latter_reader_txnid;
  if (env->lck_mmap.lck) {
    for (size_t i = 0; i < out->mi_numreaders; ++i) {
      const mdbx_pid_t pid = atomic_load_pid(&lck->rdt[i].pid, mo_AcquireRelease);
      if (pid) {
        const txnid_t txnid = safe64_read(&lck->rdt[i].txnid);
        if (overall_latter_reader_txnid > txnid)
          overall_latter_reader_txnid = txnid;
        if (pid == env->registered_reader_pid && self_latter_reader_txnid > txnid)
          self_latter_reader_txnid = txnid;
      }
    }
  }
  out->mi_self_latter_reader_txnid = self_latter_reader_txnid;
  out->mi_latter_reader_txnid = overall_latter_reader_txnid;

  osal_compiler_barrier();
  return MDBX_SUCCESS;
}

__cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, troika_t *troika) {
  MDBX_envinfo snap;
  int rc = env_info_snap(env, txn, &snap, troika);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  while (1) {
    rc = env_info_snap(env, txn, out, troika);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    snap.mi_since_sync_seconds16dot16 = out->mi_since_sync_seconds16dot16;
    snap.mi_since_reader_check_seconds16dot16 = out->mi_since_reader_check_seconds16dot16;
    if (likely(memcmp(&snap, out, sizeof(MDBX_envinfo)) == 0))
      return MDBX_SUCCESS;
    memcpy(&snap, out, sizeof(MDBX_envinfo));
  }
}

__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *arg, size_t bytes) {
  if (unlikely((env == nullptr && txn == nullptr) || arg == nullptr))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(bytes != sizeof(MDBX_envinfo)))
    return LOG_IFERR(MDBX_EINVAL);

  if (txn) {
    int err = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
    if (unlikely(err != MDBX_SUCCESS))
      return LOG_IFERR(err);
  }
  if (env) {
    int err = check_env(env, false);
    if (unlikely(err != MDBX_SUCCESS))
      return LOG_IFERR(err);
    if (txn && unlikely(txn->env != env))
      return LOG_IFERR(MDBX_EINVAL);
  } else {
    env = txn->env;
  }

  troika_t troika;
  return LOG_IFERR(env_info(env, txn, arg, &troika));
}

__cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out, size_t bytes) {
#if defined(_WIN32) || defined(_WIN64)
  wchar_t *pathnameW = nullptr;
  int rc = osal_mb2w(pathname, &pathnameW);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = mdbx_preopen_snapinfoW(pathnameW, out, bytes);
    osal_free(pathnameW);
  }
  return LOG_IFERR(rc);
}

__cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, size_t bytes) {
#endif /* Windows */
  if (unlikely(!out))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(bytes != sizeof(MDBX_envinfo)))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) {
    ERROR("unsuitable system pagesize %u", globals.sys_pagesize);
    return LOG_IFERR(MDBX_INCOMPATIBLE);
  }

  memset(out, 0, bytes);
  MDBX_env env;
  memset(&env, 0, sizeof(env));
  env.pid = osal_getpid();
  env.flags = MDBX_RDONLY | MDBX_NORDAHEAD | MDBX_ACCEDE | MDBX_VALIDATION;
  env.stuck_meta = -1;
  env.lck_mmap.fd = INVALID_HANDLE_VALUE;
  env.lazy_fd = INVALID_HANDLE_VALUE;
  env.dsync_fd = INVALID_HANDLE_VALUE;
  env.fd4meta = INVALID_HANDLE_VALUE;
#if defined(_WIN32) || defined(_WIN64)
  env.dxb_lock_event = INVALID_HANDLE_VALUE;
  env.lck_lock_event = INVALID_HANDLE_VALUE;
  env.ioring.overlapped_fd = INVALID_HANDLE_VALUE;
#endif /* Windows */
  env_options_init(&env);

  int err, rc = env_handle_pathname(&env, pathname, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;
  rc = osal_openfile(MDBX_OPEN_DXB_READ, &env, env.pathname.dxb, &env.lazy_fd, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

  meta_t header;
  rc = dxb_read_header(&env, &header, 0, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

  out->mi_dxb_pagesize = env_setup_pagesize(&env, header.pagesize);
  out->mi_geo.lower = pgno2bytes(&env, header.geometry.lower);
  out->mi_geo.upper = pgno2bytes(&env, header.geometry.upper);
  out->mi_geo.shrink = pgno2bytes(&env, pv2pages(header.geometry.shrink_pv));
  out->mi_geo.grow = pgno2bytes(&env, pv2pages(header.geometry.grow_pv));
  out->mi_geo.current = pgno2bytes(&env, header.geometry.now);
  out->mi_last_pgno = header.geometry.first_unallocated - 1;

  out->mi_recent_txnid = constmeta_txnid(&header);
  const unsigned n = 0;
  out->mi_meta_sign[n] = unaligned_peek_u64(4, &header.sign);
  memcpy(&out->mi_bootid.meta[n], &header.bootid, 16);
  memcpy(&out->mi_dxbid, &header.dxbid, 16);

bailout:
  err = env_info_sys(&env, out);
  rc = rc ? rc : err;
  err = env_close(&env, false);
  rc = rc ? rc : err;
  return LOG_IFERR(rc);
}

/*----------------------------------------------------------------------------*/

__cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper,
                                 intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize) {
  int rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  MDBX_txn *const txn_owned = env_owned_wrtxn(env);
  bool should_unlock = false;

#if MDBX_DEBUG > 0 && 0 /* минимальные шаги для проверки/отладки уже не нужны */
  if (growth_step < 0) {
    growth_step = 1;
    if (shrink_threshold < 0)
      shrink_threshold = 1;
  }
#endif /* MDBX_DEBUG > 0 */

  if (env->dxb_mmap.base) {
    /* env already mapped */
    if (unlikely(env->flags & MDBX_RDONLY))
      return LOG_IFERR(MDBX_EACCESS);

    if (!txn_owned) {
      int err = lck_txn_lock(env, false);
      if (unlikely(err != MDBX_SUCCESS))
        return LOG_IFERR(err);
      should_unlock = true;
      env->basal_txn->wr.troika = meta_tap(env);
      eASSERT0(env, !env->txn && !env->basal_txn->nested);
      env->basal_txn->txnid = env->basal_txn->wr.troika.txnid[env->basal_txn->wr.troika.recent];
      txn_gc_detent(env->basal_txn);
    }

    /* get untouched params from current TXN or DB */
    if (pagesize <= 0 || pagesize >= INT_MAX)
      pagesize = env->ps;
    if (pagesize != (intptr_t)env->ps) {
      rc = MDBX_EINVAL;
      goto bailout;
    }

    const geo_t *const geo = env->txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->wr.troika).ptr_c->geometry;
    if (size_lower < 0)
      size_lower =
          MDBX_ROUNDING_TO_ALLOCATION_GRANULARITY ? pgno_ceil2os_bytes(env, geo->lower) : pgno2bytes(env, geo->lower);
    if (size_now < 0)
      size_now = pgno_ceil2os_bytes(env, geo->now);
    if (size_upper < 0)
      size_upper = pgno_ceil2os_bytes(env, geo->upper);
    if (growth_step < 0)
      growth_step = pgno2bytes(env, pv2pages(geo->grow_pv));
    if (shrink_threshold < 0)
      shrink_threshold = pgno2bytes(env, pv2pages(geo->shrink_pv));

    const size_t usedbytes = pgno_ceil2os_bytes(env, mvcc_snapshot_largest(env, geo->first_unallocated));
    if ((size_t)size_upper < usedbytes) {
      rc = MDBX_MAP_FULL;
      goto bailout;
    }
    if ((size_t)size_now < usedbytes)
      size_now = usedbytes;
  } else {
    /* env NOT yet mapped */
    if (unlikely(env->txn))
      return LOG_IFERR(MDBX_PANIC);

    /* is requested some auto-value for pagesize ? */
    if (pagesize >= INT_MAX /* maximal */)
      pagesize = MDBX_MAX_PAGESIZE;
    else if (pagesize == 0 /* minimal */)
      pagesize = MDBX_MIN_PAGESIZE;
    else if (pagesize < 0 /* default */) {
      pagesize = globals.sys_pagesize;
      if ((uintptr_t)pagesize > MDBX_MAX_PAGESIZE)
        pagesize = MDBX_MAX_PAGESIZE;
      eASSERT0(env, (uintptr_t)pagesize >= MDBX_MIN_PAGESIZE);

      /* choose pagesize */
      intptr_t top = (size_now > size_lower) ? size_now : size_lower;
      if (size_upper > top)
        top = size_upper;
      if (top < 0 /* default */)
        top = reasonable_db_maxsize();
      else if (top == 0 /* minimal */)
        top = MIN_MAPSIZE;
      else if (top >= (intptr_t)MAX_MAPSIZE /* maximal */)
        top = MAX_MAPSIZE;

      while (top > pagesize * (int64_t)(MAX_PAGENO + 1) && pagesize < MDBX_MAX_PAGESIZE)
        pagesize <<= 1;
    }
  }

  if (pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2(pagesize)) {
    rc = MDBX_EINVAL;
    goto bailout;
  }

  const bool size_lower_default = size_lower < 0;
  if (size_lower <= 0) {
    size_lower = (size_lower == 0) ? MIN_MAPSIZE : pagesize * MDBX_WORDBITS;
    if (size_lower / pagesize < MIN_PAGENO)
      size_lower = MIN_PAGENO * pagesize;
  }
  if (size_lower >= INTPTR_MAX) {
    size_lower = reasonable_db_maxsize();
    if ((size_t)size_lower / pagesize > MAX_PAGENO + 1)
      size_lower = pagesize * (MAX_PAGENO + 1);
  }

  if (size_now >= INTPTR_MAX) {
    size_now = reasonable_db_maxsize();
    if ((size_t)size_now / pagesize > MAX_PAGENO + 1)
      size_now = pagesize * (MAX_PAGENO + 1);
  }

  if (size_upper <= 0) {
    if ((growth_step == 0 || size_upper == 0) && size_now >= size_lower)
      size_upper = size_now;
    else if (size_now <= 0 || size_now >= reasonable_db_maxsize() / 2)
      size_upper = reasonable_db_maxsize();
    else if ((size_t)size_now >= MAX_MAPSIZE32 / 2 && (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3)
      size_upper = MAX_MAPSIZE32;
    else {
      size_upper = ceil_powerof2(((size_t)size_now < MAX_MAPSIZE / 4) ? size_now + size_now : size_now + size_now / 2,
                                 MEGABYTE * MDBX_WORDBITS * MDBX_WORDBITS / 32);
      if ((size_t)size_upper > MAX_MAPSIZE)
        size_upper = MAX_MAPSIZE;
    }
    if ((size_t)size_upper / pagesize > (MAX_PAGENO + 1))
      size_upper = pagesize * (MAX_PAGENO + 1);
  } else if (size_upper >= INTPTR_MAX) {
    size_upper = reasonable_db_maxsize();
    if ((size_t)size_upper / pagesize > MAX_PAGENO + 1)
      size_upper = pagesize * (MAX_PAGENO + 1);
  }

  if (unlikely(size_lower < (intptr_t)MIN_MAPSIZE || size_lower > size_upper)) {
    /* паранойа на случай переполнения при невероятных значениях */
    rc = MDBX_EINVAL;
    goto bailout;
  }

  if (size_now <= 0) {
    size_now = size_lower;
    if (size_upper >= size_lower && size_now > size_upper)
      size_now = size_upper;
  }

  if ((uint64_t)size_lower / pagesize < MIN_PAGENO) {
    size_lower = pagesize * MIN_PAGENO;
    if (unlikely(size_lower > size_upper)) {
      /* паранойа на случай переполнения при невероятных значениях */
      rc = MDBX_EINVAL;
      goto bailout;
    }
    if (size_now < size_lower)
      size_now = size_lower;
  }

  if (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
    rc = MDBX_TOO_LARGE;
    goto bailout;
  }

  const size_t unit_ps = (globals.sys_pagesize > (size_t)pagesize) ? globals.sys_pagesize : (size_t)pagesize;
  const size_t unit_ag = (globals.sys_allocation_granularity > unit_ps &&
                          (growth_step < 0 || (size_t)growth_step >= globals.sys_allocation_granularity))
                             ? globals.sys_allocation_granularity
                             : unit_ps;
  size_lower = ceil_powerof2(size_lower, unit_ps);
  size_upper = ceil_powerof2(size_upper, unit_ag);
  size_now = ceil_powerof2(size_now, unit_ag);

  /* LY: подбираем значение size_upper:
   *  - кратное размеру unit_ag (размеру страницы БД и системному размеру выделения)
   *  - без нарушения MAX_MAPSIZE и MAX_PAGENO */
  while (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
    if ((size_t)size_upper < unit_ag + MIN_MAPSIZE || (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) {
      /* паранойа на случай переполнения при невероятных значениях */
      rc = MDBX_EINVAL;
      goto bailout;
    }
    size_upper -= unit_ag;
    if ((size_t)size_upper < (size_t)size_lower)
      size_lower = size_upper;
  }
  eASSERT0(env, (size_upper - size_lower) % globals.sys_pagesize == 0);

  if (size_now < size_lower)
    size_now = size_lower;
  if (size_now > size_upper)
    size_now = size_upper;

  if (growth_step < 0) {
    growth_step = ((size_t)(size_upper - size_lower)) / 42;
    if (!size_lower_default && growth_step > size_lower && size_lower < (intptr_t)MEGABYTE)
      growth_step = size_lower;
    else if (growth_step / size_lower > 64)
      growth_step = size_lower << 6;
    if (growth_step < 65536)
      growth_step = 65536;
    if ((size_upper - size_lower) / growth_step > 65536)
      growth_step = (size_upper - size_lower) >> 16;
    const intptr_t growth_step_limit = MEGABYTE * ((MDBX_WORDBITS > 32) ? 4096 : 256);
    if (growth_step > growth_step_limit)
      growth_step = growth_step_limit;
  }
  if (growth_step == 0 && shrink_threshold > 0)
    growth_step = 1;
  growth_step = ceil_powerof2(growth_step, unit_ag);

  if (shrink_threshold < 0)
    shrink_threshold = growth_step + growth_step;
  shrink_threshold = ceil_powerof2(shrink_threshold, unit_ag);

  //----------------------------------------------------------------------------

  if (!env->dxb_mmap.base) {
    /* save user's geo-params for future open/create */
    if (pagesize != (intptr_t)env->ps)
      env_setup_pagesize(env, pagesize);
    env->geo_in_bytes.lower = size_lower;
    env->geo_in_bytes.now = size_now;
    env->geo_in_bytes.upper = size_upper;
    env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step))));
    env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold))));
    env_options_adjust_defaults(env);

    ENSURE_OBJ(env, env->geo_in_bytes.lower >= MIN_MAPSIZE);
    ENSURE_OBJ(env, env->geo_in_bytes.lower / (unsigned)pagesize >= MIN_PAGENO);
    ENSURE_OBJ(env, env->geo_in_bytes.lower % (unsigned)pagesize == 0);
    ENSURE_OBJ(env, env->geo_in_bytes.lower % globals.sys_pagesize == 0);

    ENSURE_OBJ(env, env->geo_in_bytes.upper <= MAX_MAPSIZE);
    ENSURE_OBJ(env, env->geo_in_bytes.upper / (unsigned)pagesize <= MAX_PAGENO + 1);
    ENSURE_OBJ(env, env->geo_in_bytes.upper % (unsigned)pagesize == 0);
    ENSURE_OBJ(env, env->geo_in_bytes.upper % globals.sys_pagesize == 0);

    ENSURE_OBJ(env, env->geo_in_bytes.now >= env->geo_in_bytes.lower);
    ENSURE_OBJ(env, env->geo_in_bytes.now <= env->geo_in_bytes.upper);
    ENSURE_OBJ(env, env->geo_in_bytes.now % (unsigned)pagesize == 0);
    ENSURE_OBJ(env, env->geo_in_bytes.now % globals.sys_pagesize == 0);

    ENSURE_OBJ(env, env->geo_in_bytes.grow % (unsigned)pagesize == 0);
    ENSURE_OBJ(env, env->geo_in_bytes.grow % globals.sys_pagesize == 0);
    ENSURE_OBJ(env, env->geo_in_bytes.shrink % (unsigned)pagesize == 0);
    ENSURE_OBJ(env, env->geo_in_bytes.shrink % globals.sys_pagesize == 0);

    rc = MDBX_SUCCESS;
  } else {
    /* apply new params to opened environment */
    ENSURE_OBJ(env, pagesize == (intptr_t)env->ps);
    meta_t meta;
    memset(&meta, 0, sizeof(meta));
    if (!env->txn) {
      const meta_ptr_t head = meta_recent(env, &env->basal_txn->wr.troika);

      uint64_t timestamp = 0;
      while ("workaround for "
             "https://libmdbx.dqdkfa.ru/dead-github/issues/269") {
        rc = coherency_fetch_head(env->basal_txn, head, &timestamp);
        if (likely(rc == MDBX_SUCCESS))
          break;
        if (unlikely(rc != MDBX_RESULT_TRUE))
          goto bailout;
      }
      meta = *head.ptr_c;
      const txnid_t txnid = safe64_txnid_next(head.txnid);
      if (unlikely(txnid > MAX_TXNID)) {
        rc = MDBX_TXN_FULL;
        ERROR("txnid overflow, raise %d", rc);
        goto bailout;
      }
      meta_set_txnid(env, &meta, txnid);
    }

    const geo_t *const current_geo = &(env->txn ? env->txn : env->basal_txn)->geo;
    /* update env-geo to avoid influences */
    env->geo_in_bytes.now = pgno2bytes(env, current_geo->now);
    env->geo_in_bytes.lower = pgno2bytes(env, current_geo->lower);
    env->geo_in_bytes.upper = pgno2bytes(env, current_geo->upper);
    env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(current_geo->grow_pv));
    env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(current_geo->shrink_pv));

    geo_t new_geo;
    new_geo.lower = bytes2pgno(env, size_lower);
    new_geo.now = bytes2pgno(env, size_now);
    new_geo.upper = bytes2pgno(env, size_upper);
    new_geo.grow_pv = pages2pv(bytes2pgno(env, growth_step));
    new_geo.shrink_pv = pages2pv(bytes2pgno(env, shrink_threshold));
    new_geo.first_unallocated = current_geo->first_unallocated;

    ENSURE_OBJ(env, pgno_ceil2sp_bytes(env, new_geo.lower) == (size_t)size_lower);
    ENSURE_OBJ(env, pgno_ceil2sp_bytes(env, new_geo.upper) == (size_t)size_upper);
    ENSURE_OBJ(env, pgno_ceil2sp_bytes(env, new_geo.now) == (size_t)size_now);
    ENSURE_OBJ(env, new_geo.grow_pv == pages2pv(pv2pages(new_geo.grow_pv)));
    ENSURE_OBJ(env, new_geo.shrink_pv == pages2pv(pv2pages(new_geo.shrink_pv)));

    ENSURE_OBJ(env, (size_t)size_lower >= MIN_MAPSIZE);
    ENSURE_OBJ(env, new_geo.lower >= MIN_PAGENO);
    ENSURE_OBJ(env, (size_t)size_upper <= MAX_MAPSIZE);
    ENSURE_OBJ(env, new_geo.upper <= MAX_PAGENO + 1);
    ENSURE_OBJ(env, new_geo.now >= new_geo.first_unallocated);
    ENSURE_OBJ(env, new_geo.upper >= new_geo.now);
    ENSURE_OBJ(env, new_geo.now >= new_geo.lower);

    if (memcmp(current_geo, &new_geo, sizeof(geo_t)) != 0) {
#if defined(_WIN32) || defined(_WIN64)
      /* Was DB shrinking disabled before and now it will be enabled? */
      if (new_geo.lower < new_geo.upper && new_geo.shrink_pv &&
          !(current_geo->lower < current_geo->upper && current_geo->shrink_pv)) {
        if (!env->lck_mmap.lck) {
          rc = MDBX_EPERM;
          goto bailout;
        }
        int err = lck_rdt_lock(env);
        if (unlikely(MDBX_IS_ERROR(err))) {
          rc = err;
          goto bailout;
        }

        /* Check if there are any reading threads that do not use the SRWL */
        const size_t CurrentTid = GetCurrentThreadId();
        const reader_slot_t *const begin = env->lck_mmap.lck->rdt;
        const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease);
        for (const reader_slot_t *reader = begin; reader < end; ++reader) {
          if (reader->pid.weak == env->pid && reader->tid.weak != CurrentTid &&
              reader->tid.weak < MDBX_TID_TXN_OUSTED) {
            /* At least one thread may don't use SRWL */
            rc = MDBX_EPERM;
            break;
          }
        }

        lck_rdt_unlock(env);
        if (unlikely(rc != MDBX_SUCCESS))
          goto bailout;
      }
#endif /* Windows */

      if (new_geo.now != current_geo->now || new_geo.upper != current_geo->upper) {
        rc = dxb_resize(env, current_geo->first_unallocated, new_geo.now, new_geo.upper, explicit_resize);
        if (unlikely(rc != MDBX_SUCCESS))
          goto bailout;
      }
      if (env->txn) {
        env->txn->geo = new_geo;
        env->txn->flags |= MDBX_TXN_DIRTY;
      } else {
        meta.geometry = new_geo;
        rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->wr.troika);
        if (likely(rc == MDBX_SUCCESS)) {
          env->geo_in_bytes.now = pgno2bytes(env, new_geo.now = meta.geometry.now);
          env->geo_in_bytes.upper = pgno2bytes(env, new_geo.upper = meta.geometry.upper);
        }
      }
    }
    if (likely(rc == MDBX_SUCCESS)) {
      /* update env-geo to avoid influences */
      eASSERT0(env, env->geo_in_bytes.now == pgno2bytes(env, new_geo.now));
      env->geo_in_bytes.lower = pgno2bytes(env, new_geo.lower);
      eASSERT0(env, env->geo_in_bytes.upper == pgno2bytes(env, new_geo.upper));
      env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(new_geo.grow_pv));
      env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(new_geo.shrink_pv));
    }
  }

bailout:
  if (should_unlock)
    lck_txn_unlock(env);
  return LOG_IFERR(rc);
}

__cold int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return LOG_IFERR(env_sync(env, force, nonblock));
}

__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_stat *dest, size_t bytes) {
  if (unlikely(!dest))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(bytes != sizeof(MDBX_stat)))
    return LOG_IFERR(MDBX_EINVAL);

  if (likely(txn)) {
    if (env && unlikely(txn->env != env))
      return LOG_IFERR(MDBX_EINVAL);
    int rc = check_txn(txn, MDBX_TXN_BLOCKED);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);

    return LOG_IFERR(tbl_stat_summary(txn, dest));
  }

  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  MDBX_txn *txn_owned = env_owned_wrtxn(env);
  if (txn_owned)
    /* inside write-txn */
    return LOG_IFERR(tbl_stat_summary(txn_owned, dest));

  rc = mdbx_txn_begin((MDBX_env *)env, nullptr, MDBX_TXN_RDONLY, &txn_owned);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = tbl_stat_summary(txn_owned, dest);
  int err = txn_abort(txn_owned, nullptr);
  rc = (rc == MDBX_SUCCESS && err != rc) ? err : rc;
  return LOG_IFERR(rc);
}

/*------------------------------------------------------------------------------
 * Readers API */

__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func func, void *ctx) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!func))
    return LOG_IFERR(MDBX_EINVAL);

  rc = MDBX_RESULT_TRUE;
  int serial = 0;
  lck_t *const lck = env->lck_mmap.lck;
  if (likely(lck)) {
    const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
    for (size_t i = 0; i < snap_nreaders; i++) {
      const reader_slot_t *slot = lck->rdt + i;
    retry_reader:;
      const mdbx_pid_t pid = atomic_load_pid(&slot->pid, mo_AcquireRelease);
      if (!pid)
        continue;
      txnid_t txnid = safe64_read(&slot->txnid);
      const uint64_t tid = atomic_load64(&slot->tid, mo_Relaxed);
      const pgno_t pages_used = atomic_load32(&slot->snapshot_pages_used, mo_Relaxed);
      const uint64_t reader_pages_retired = atomic_load64(&slot->snapshot_pages_retired, mo_Relaxed);
      if (unlikely(txnid != safe64_read(&slot->txnid) || pid != atomic_load_pid(&slot->pid, mo_AcquireRelease) ||
                   tid != atomic_load64(&slot->tid, mo_Relaxed) ||
                   pages_used != atomic_load32(&slot->snapshot_pages_used, mo_Relaxed) ||
                   reader_pages_retired != atomic_load64(&slot->snapshot_pages_retired, mo_Relaxed)))
        goto retry_reader;

      eASSERT0(env, txnid > 0);
      if (txnid >= SAFE64_INVALID_THRESHOLD)
        txnid = 0;

      size_t bytes_used = 0;
      size_t bytes_retained = 0;
      uint64_t lag = 0;
      if (txnid) {
        troika_t troika = meta_tap(env);
      retry_header:;
        const meta_ptr_t head = meta_recent(env, &troika);
        const uint64_t head_pages_retired = unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired);
        if (unlikely(meta_should_retry(env, &troika) ||
                     head_pages_retired != unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired)))
          goto retry_header;

        lag = (head.txnid - txnid) / xMDBX_TXNID_STEP;
        bytes_used = pgno2bytes(env, pages_used);
        bytes_retained = (head_pages_retired > reader_pages_retired)
                             ? pgno2bytes(env, (pgno_t)(head_pages_retired - reader_pages_retired))
                             : 0;
      }
      rc = func(ctx, ++serial, (unsigned)i, pid, (mdbx_tid_t)((intptr_t)tid), txnid, lag, bytes_used, bytes_retained);
      if (unlikely(rc != MDBX_SUCCESS))
        break;
    }
  }

  return LOG_IFERR(rc);
}

__cold int mdbx_reader_check(MDBX_env *env, int *dead) {
  if (dead)
    *dead = 0;
  return LOG_IFERR(mvcc_cleanup_dead(env, false, dead));
}

__cold int mdbx_thread_register(const MDBX_env *env) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!env->lck_mmap.lck))
    return LOG_IFERR((env->flags & MDBX_EXCLUSIVE) ? MDBX_EINVAL : MDBX_EPERM);

  if (unlikely((env->flags & ENV_TXKEY) == 0)) {
    eASSERT0(env, env->flags & MDBX_NOSTICKYTHREADS);
    return LOG_IFERR(MDBX_EINVAL) /* MDBX_NOSTICKYTHREADS mode */;
  }

  eASSERT0(env, (env->flags & (MDBX_NOSTICKYTHREADS | ENV_TXKEY)) == ENV_TXKEY);
  reader_slot_t *slot = thread_rthc_get(env->me_txkey);
  if (unlikely(slot != nullptr)) {
    eASSERT0(env, atomic_load_pid(&slot->pid, mo_Relaxed) == env->registered_reader_pid);
    eASSERT1(env, env->registered_reader_pid == osal_getpid());
    eASSERT1(env, slot->tid.weak == osal_thread_self());
    if (unlikely(atomic_load_pid(&slot->pid, mo_Relaxed) != env->registered_reader_pid))
      return LOG_IFERR(MDBX_BAD_RSLOT);
    return MDBX_RESULT_TRUE /* already registered */;
  }

  return LOG_IFERR(mvcc_bind_slot((MDBX_env *)env).err);
}

__cold int mdbx_thread_unregister(const MDBX_env *env) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!env->lck_mmap.lck))
    return MDBX_RESULT_TRUE;

  if (unlikely((env->flags & ENV_TXKEY) == 0)) {
    eASSERT0(env, env->flags & MDBX_NOSTICKYTHREADS);
    return MDBX_RESULT_TRUE /* MDBX_NOSTICKYTHREADS mode */;
  }

  eASSERT0(env, (env->flags & (MDBX_NOSTICKYTHREADS | ENV_TXKEY)) == ENV_TXKEY);
  reader_slot_t *slot = thread_rthc_get(env->me_txkey);
  if (unlikely(slot == nullptr))
    return MDBX_RESULT_TRUE /* not registered */;

  eASSERT0(env, atomic_load_pid(&slot->pid, mo_Relaxed) == env->registered_reader_pid);
  if (unlikely(atomic_load_pid(&slot->pid, mo_Relaxed) != env->registered_reader_pid ||
               slot->tid.weak != osal_thread_self()))
    return LOG_IFERR(MDBX_BAD_RSLOT);

  eASSERT0(env, slot->txnid.weak >= SAFE64_INVALID_THRESHOLD);
  if (unlikely(slot->txnid.weak < SAFE64_INVALID_THRESHOLD))
    return LOG_IFERR(MDBX_BUSY) /* transaction is still active */;

  atomic_store32(&slot->pid, 0, mo_Relaxed);
  atomic_store32(&env->lck->rdt_refresh_flag, true, mo_AcquireRelease);
  thread_rthc_set(env->me_txkey, nullptr);
  return MDBX_SUCCESS;
}

/*------------------------------------------------------------------------------
 * Locking API */

int mdbx_txn_lock(MDBX_env *env, bool dont_wait) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(env->flags & MDBX_RDONLY))
    return LOG_IFERR(MDBX_EACCESS);
  if (dont_wait && unlikely(env->basal_txn->owner || (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0))
    return LOG_IFERR(MDBX_BUSY);

  return LOG_IFERR(lck_txn_lock(env, dont_wait));
}

int mdbx_txn_unlock(MDBX_env *env) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(env->flags & MDBX_RDONLY))
    return LOG_IFERR(MDBX_EACCESS);
#if MDBX_TXN_CHECKOWNER
  if (unlikely(env->basal_txn->owner != osal_thread_self()))
    return LOG_IFERR(MDBX_THREAD_MISMATCH);
#endif /* MDBX_TXN_CHECKOWNER */
  if (unlikely((env->basal_txn->flags & MDBX_TXN_FINISHED) == 0))
    return LOG_IFERR(MDBX_BUSY);

  lck_txn_unlock(env);
  return MDBX_SUCCESS;
}

/*------------------------------------------------------------------------------
 * Auxiliary */

__cold const char *mdbx_ratio2digits(uint64_t numerator, uint64_t denominator, int precision, char *buffer,
                                     size_t buffer_size) {
  if (!buffer)
    return "nullptr";
  else if (buffer_size < sizeof(ratio2digits_buffer_t))
    return "buffer-to-small";
  else if (!denominator)
    return numerator ? "infinity" : "undefined";
  else
    return ratio2digits(numerator, denominator, (ratio2digits_buffer_t *)buffer, precision);
}

__cold const char *mdbx_ratio2percents(uint64_t value, uint64_t whole, char *buffer, size_t buffer_size) {
  if (!buffer)
    return "nullptr";
  else if (buffer_size < sizeof(ratio2digits_buffer_t))
    return "buffer-to-small";
  else if (!whole)
    return value ? "infinity" : "undefined";
  else
    return ratio2percent(value, whole, (ratio2digits_buffer_t *)buffer);
}

__cold int mdbx_gc_info(MDBX_txn *txn, MDBX_gc_info_t *info, size_t bytes, MDBX_gc_iter_func iter_func,
                        void *iter_ctx) {
  if (unlikely(!info || bytes != sizeof(MDBX_gc_info_t)))
    return MDBX_EINVAL;
  memset(info, 0, sizeof(*info));

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  info->pages_total = txn->geo.upper;
  info->pages_backed = txn->geo.end_pgno;
  info->pages_allocated = txn->geo.first_unallocated;

  const volatile lck_t *const lck = txn->env->lck;
  if (lck) {
    do {
      info->max_reader_lag = lck->pgops.gc_prof.max_reader_lag;
      info->max_retained_pages = lck->pgops.gc_prof.max_retained_pages;
      osal_memory_fence(mo_AcquireRelease, false);
    } while (unlikely(info->max_reader_lag != lck->pgops.gc_prof.max_reader_lag ||
                      info->max_retained_pages != lck->pgops.gc_prof.max_retained_pages));
  }

  const tree_t *const gc = &txn->dbs[FREE_DBI];
  info->pages_gc += gc->branch_pages;
  info->pages_gc += gc->leaf_pages;
  info->pages_gc += gc->large_pages;

  const txnid_t reclaiming_detent = (txn->flags & txn_ro_flat) ? mvcc_shapshot_oldest_ro(txn, false).oldest_txnid
                                                               : mvcc_shapshot_oldest_rw(txn).oldest_txnid;
  if ((txn->flags & txn_ro_flat) == 0) {
    const size_t workset = txn->wr.loose_count + pnl_size(txn->wr.repnl);
    info->gc_reclaimable.pages += workset;
    info->gc_reclaimable.pages += info->pages_gc;
    info->pages_gc += workset + pnl_size(txn->wr.retired_pages) - /* retired_stored */ 0;
  }

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, FREE_DBI);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  MDBX_val gc_key, gc_data;
  rc = outer_first(&cx.outer, &gc_key, &gc_data);
  if (rc == MDBX_SUCCESS) {
    cx.outer.next = txn->cursors[FREE_DBI];
    txn->cursors[FREE_DBI] = &cx.outer;
    for (;;) {
      if (unlikely(gc_check_keylen(gc_key.iov_len))) {
        ERROR("%s/%d: %s", "corrupted GC-record", rc = MDBX_CORRUPTED, gc_check_keylen(gc_key.iov_len));
        break;
      }

      const glr_t glr = gc_row_pnl(txn, gc_data);
      if (unlikely(glr.err != MDBX_SUCCESS)) {
        ERROR("%s/%d: %s", "corrupted GC-record", rc = glr.err, glr.reason);
        break;
      }

      const txnid_t id = unaligned_peek_u64(4, gc_key.iov_base);
      if ((txn->flags & txn_ro_flat) == 0 && gc_is_reclaimed(txn, id))
        continue;

      const size_t len = pnl_size(glr.pnl);
      const bool is_reclaimable = id <= reclaiming_detent;
      info->pages_gc += len;
      if (is_reclaimable)
        info->gc_reclaimable.pages += len;
      for (size_t span, i = 1; i <= len; i += span) {
        span = pnl_scan_span(glr.pnl, i);
        const size_t pgno = glr.pnl[i];
        if (is_reclaimable) {
          histogram_acc(span, &info->gc_reclaimable.span_histogram);
          histogram_acc(pgno, &info->gc_reclaimable.pgno_distribution);
        }
        if (iter_func) {
          rc = iter_func(iter_ctx, txn, id, pgno, span, is_reclaimable);
          if (rc != MDBX_RESULT_FALSE)
            break;
        }
      }

      rc = outer_next(&cx.outer, &gc_key, &gc_data, MDBX_NEXT);
      if (rc != MDBX_SUCCESS) {
        rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc;
        break;
      }
    }
    txn->cursors[FREE_DBI] = cx.outer.next;
  }
  return LOG_IFERR(rc);
}

__cold int mdbx_env_defrag(MDBX_env *env, size_t defrag_atleast, size_t time_atleast_dot16, size_t defrag_enough,
                           size_t time_limit_dot16, intptr_t acceptable_backlash, intptr_t preferred_batch,
                           MDBX_defrag_notify_func progress_callback, void *ctx, MDBX_defrag_result_t *result) {
  if (result)
    memset(result, 0, sizeof(*result));
  if (unlikely(defrag_enough < defrag_atleast) && defrag_enough)
    return LOG_IFERR(MDBX_EINVAL);
  if (unlikely(time_limit_dot16 < time_atleast_dot16) && time_limit_dot16)
    return LOG_IFERR(MDBX_EINVAL);
  if (unlikely(!progress_callback && ctx))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(env->flags & MDBX_RDONLY))
    return LOG_IFERR(MDBX_EACCESS);

  dfc_t dfc;
  MDBX_txn *txn = env_owned_wrtxn(env);
  if (txn) {
    rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
    if (rc == MDBX_SUCCESS && txn->parent)
      rc = MDBX_BAD_TXN;
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  } else {
    rc = txn_basal_start(txn = env->basal_txn, MDBX_TXN_READWRITE | MDBX_TXN_NOWEAKING);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    txn->signature = txn_signature;
    txn->userctx = &dfc;
  }

  if (acceptable_backlash < 0)
    acceptable_backlash = CURSOR_STACK_SIZE;
  else
    acceptable_backlash = ((size_t)acceptable_backlash < txn->env->maxgc_large1page) ? (size_t)acceptable_backlash
                                                                                     : txn->env->maxgc_large1page;

  rc = defrag_init(&dfc, txn, defrag_atleast, time_atleast_dot16, defrag_enough, time_limit_dot16, preferred_batch);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  dfc.user_ctx = ctx;
  dfc.user_callback = progress_callback;

  while ((size_t)acceptable_backlash + dfc.payload_pages < txn->geo.first_unallocated && !defrag_discontinued(&dfc)) {
    const pgno_t prev_stumble = dfc.stumble_pgno;
    rc = defrag_cycle(&dfc);

#if MDBX_CHECKING > 1
    pnl_free(dfc.repnl_clone);
    dfc.repnl_clone = nullptr;
#endif /* MDBX_CHECKING > 1 */

    if (MDBX_IS_ERROR(rc)) {
      txn->flags |= MDBX_TXN_ERROR;
      dfc.stopping_reasons |= MDBX_defrag_error;
      break;
    }

    dfc.stumble_retry = (dfc.stumble_pgno && dfc.stumble_pgno == prev_stumble) ? dfc.stumble_retry + 1 : 0;
    if (dfc.stumble_retry > 3) {
      NOTICE("bailout since stucked (unable to move) at %u in a few retries", dfc.stumble_pgno);
      rc = txn->dbs[FREE_DBI].items ? MDBX_LAGGARD_READER : MDBX_RESULT_TRUE;
      break;
    }

    if ((dfc.stopping_reasons & MDBX_defrag_laggard_reader) != 0 && txn_gc_detent(txn))
      dfc.stopping_reasons -= MDBX_defrag_laggard_reader;

    if (rc != MDBX_SUCCESS) {
      if ((dfc.lp_reserve && pnl_size(dfc.lp_reserve)) || dfc.cycle_pages_scheduled != dfc.cycle_pages_moved)
        txn->flags |= MDBX_TXN_ERROR;
      break;
    } else {
      ASSERT(!dfc.lp_reserve || pnl_size(dfc.lp_reserve) == 0);
      ASSERT(dfc.cycle_pages_scheduled == dfc.cycle_pages_moved);
    }

    rc = txn_basal_checkpoint(txn, MDBX_TXN_NOMETASYNC, nullptr);
    if (unlikely(rc != MDBX_SUCCESS)) {
      dfc.txn = txn = nullptr;
      dfc.stopping_reasons |= MDBX_defrag_error;
      break;
    }
  }

  if (txn && txn->userctx == &dfc) {
    if ((txn->flags & MDBX_TXN_ERROR) == 0) {
      if (defrag_score(&dfc, dfc.last_allocated = txn->geo.first_unallocated) > dfc.cycle_initial_score) {
        rc = txn_basal_commit(txn, nullptr);
        dfc.last_allocated = txn->geo.first_unallocated;
      }
    }
    dfc.txn = nullptr;
    int err = txn_basal_end(txn, true);
    rc = (err != MDBX_SUCCESS && !MDBX_IS_ERROR(rc)) ? err : rc;
  }

  defrag_milestone(&dfc);

  if (result) {
    defrag_result(&dfc, result, 0);
    result->pages_moved = dfc.total_pages_moved;
  }

  defrag_destroy(&dfc);

  if (!MDBX_IS_ERROR(rc)) {
    if (dfc.stopping_reasons)
      rc = (dfc.stopping_reasons == MDBX_defrag_laggard_reader) ? MDBX_LAGGARD_READER : MDBX_RESULT_TRUE;
    if (dfc.last_allocated <= dfc.defrag_enough)
      rc = MDBX_SUCCESS;
  }
  return LOG_IFERR(rc);
}

static MDBX_cache_result_t cache_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
                                     MDBX_cache_entry_t *entry);

static inline bool is_outside_dxb(const MDBX_txn *txn, const void *ptr) {
  const MDBX_env *env = txn->env;
  const ptrdiff_t offset = ptr_dist(ptr, env->dxb_mmap.base);
  return offset < 0 || (size_t)offset >= pgno2bytes(env, txn->geo.first_unallocated);
}

static inline bool is_not_commited(const MDBX_txn *txn, const page_t *mp) {
  cASSERT0(txn, mp >= (const page_t *)txn->env->dxb_mmap.base &&
                    mp < (const page_t *)(ptr_disp(txn->env->dxb_mmap.base,
                                                   pgno2bytes(txn->env, txn->geo.first_unallocated))));
  return mp->txnid > txn_basis_snapshot(txn);
}

MDBX_MAYBE_UNUSED static inline bool is_inside_dxb_and_commited(const MDBX_txn *txn, const void *ptr) {
  return !is_outside_dxb(txn, ptr) && !is_not_commited(txn, ptr2page(txn->env, ptr));
}

static inline MDBX_cache_result_t cache_result(int err, MDBX_cache_status_t status) {
  MDBX_cache_result_t result = {.errcode = err, .status = status};
  return result;
}

static inline MDBX_cache_result_t cache_error(int err) {
  ASSERT(err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE);
  return cache_result(err, MDBX_CACHE_ERROR);
}

static inline MDBX_cache_result_t cache_fallback(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
                                                 MDBX_cache_status_t status) {
  MDBX_cache_entry_t stub = {.last_confirmed_txnid = 0, .trunk_txnid = 0};
  MDBX_cache_result_t result = cache_get(txn, dbi, key, data, &stub);
  if (result.status > MDBX_CACHE_DIRTY)
    result.status = status;
  return result;
}

static inline bool is_afterlast(MDBX_cursor *mc, const page_t *mp, const MDBX_val *key) {
  MDBX_val last = get_key(page_node(mp, page_numkeys(mp) - 1));
  return mc->clc->k.cmp(key, &last) > 0;
}

__hot static MDBX_cache_result_t cache_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
                                           MDBX_cache_entry_t *entry) {
  DKBUF_DEBUG;
  DEBUG("===> cached-get dbi %u, key [%s], entry %p (trunk %" PRIaTXN ", last_confirmed %" PRIaTXN
        ", data offset %zu, len %u)",
        dbi, DKEY_DEBUG(key), __Wpedantic_format_voidptr(entry), entry->trunk_txnid, entry->last_confirmed_txnid,
        entry->offset, entry->length);

  if (unlikely(entry->trunk_txnid > entry->last_confirmed_txnid))
    return cache_error(LOG_IFERR(MDBX_INVALID));

  int err = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(err != MDBX_SUCCESS))
    return cache_error(LOG_IFERR(err));

  cASSERT0(txn, entry->offset || entry->length == 0);
  cASSERT0(txn, entry->last_confirmed_txnid <= MAX_TXNID);
  if (unlikely(txn->txnid < entry->trunk_txnid))
    /* the used/read MVCC-snapshot is behind the accessible MVCC-range */
    return cache_fallback(txn, dbi, key, data, MDBX_CACHE_BEHIND);

  if (likely(txn->txnid <= entry->last_confirmed_txnid)) {
    if (entry->offset) {
      /* cache hit fast-path */
      data->iov_base = ptr_disp(txn->env->dxb_mmap.base, entry->offset);
      data->iov_len = entry->length;
      tASSERT1(txn, (!entry->offset && !entry->length) || is_inside_dxb_and_commited(txn, data->iov_base));
      return cache_result(MDBX_SUCCESS, MDBX_CACHE_HIT);
    }

    if (txn->txnid == entry->last_confirmed_txnid || txn->txnid == entry->trunk_txnid) {
      data->iov_base = nullptr;
      data->iov_len = 0;
      tASSERT1(txn, (!entry->offset && !entry->length) || is_inside_dxb_and_commited(txn, data->iov_base));
      return cache_result(MDBX_NOTFOUND, MDBX_CACHE_HIT);
    }

    /* Здесь ABA-подобная проблема:
     *
     * В случае когда txn->txnid > entry->trunk_txnid и txn->txnid < entry->last_confirmed_txnid
     * возможна ситуация, когда в истории транзакции целевой ключ (и значение) отсутствовали
     * в версии entry->trunk_txnid и entry->last_confirmed_txnid, но присутствовали в MVCC-снимке txn->txnid.
     *
     * Теоретически можно подумать о том, чтобы обработать такие случаи без полного поиска. Но сначала требуется
     * понять как часто будут возникать такие ситуации на практике и стоит-ли с этим заморачиваться. */
    return cache_fallback(txn, dbi, key, data, MDBX_CACHE_UNABLE);
  }

  err = dbi_check(txn, dbi);
  if (unlikely(err != MDBX_SUCCESS))
    return cache_error(LOG_IFERR(err));

  const uint64_t committed_snapshot_txnid = txn_basis_snapshot(txn);
  txnid_t trunk_txnid = txn->front_txnid;
  if (unlikely(txn->dbi_state[dbi] & DBI_STALE)) {
    err = tbl_refresh((MDBX_txn *)txn, dbi);
    if (unlikely(err != MDBX_SUCCESS)) {
      if (err == MDBX_NOTFOUND) {
        /* the corresponding table has been deleted */
        cASSERT0(txn, !entry->offset || trunk_txnid > entry->trunk_txnid);
      not_found:
        data->iov_base = nullptr;
        data->iov_len = 0;
        MDBX_cache_status_t status = MDBX_CACHE_DIRTY;
        if (trunk_txnid <= committed_snapshot_txnid) {
          status = MDBX_CACHE_CONFIRMED;
          if (entry->offset || !entry->trunk_txnid) {
            status = MDBX_CACHE_REFRESHED;
            cASSERT0(txn, (!entry->offset && !entry->trunk_txnid) || trunk_txnid > entry->trunk_txnid);
            entry->offset = 0;
            entry->length = 0;
            entry->trunk_txnid = trunk_txnid;
          }
          entry->last_confirmed_txnid = committed_snapshot_txnid;
        }
        return cache_result(MDBX_NOTFOUND, status);
      }
      return cache_error(LOG_IFERR(err));
    }
  }

  trunk_txnid = tbl_root_txnid(txn, dbi);
  if (trunk_txnid <= entry->last_confirmed_txnid) {
    /* the corresponding table has not been changed since the last check. */
    cASSERT0(txn, (txn->dbi_state[dbi] & DBI_DIRTY) == 0);
    cASSERT0(txn, trunk_txnid == entry->trunk_txnid);
  confirmed:
    cASSERT0(txn, trunk_txnid <= committed_snapshot_txnid && trunk_txnid <= entry->last_confirmed_txnid);
    data->iov_base = entry->offset ? ptr_disp(txn->env->dxb_mmap.base, entry->offset) : nullptr;
    data->iov_len = entry->length;
    tASSERT1(txn, (!entry->offset && !entry->length) || is_inside_dxb_and_commited(txn, data->iov_base));
    if (entry->last_confirmed_txnid == committed_snapshot_txnid)
      return cache_result(data->iov_base ? MDBX_SUCCESS : MDBX_NOTFOUND, MDBX_CACHE_HIT);
    entry->last_confirmed_txnid = committed_snapshot_txnid;
    return cache_result(data->iov_base ? MDBX_SUCCESS : MDBX_NOTFOUND, MDBX_CACHE_CONFIRMED);
  }

  if (unlikely(txn->dbs[dbi].root == P_INVALID)) {
    /* the corresponding table is empty now */
    cASSERT0(txn, !entry->offset || trunk_txnid > entry->trunk_txnid);
    goto not_found;
  }

  cursor_couple_t cx;
  err = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(err != MDBX_SUCCESS))
    return cache_error(LOG_IFERR(err));

  alignkey_t aligned;
  err = check_key(&cx.outer, key, &aligned);
  if (unlikely(err != MDBX_SUCCESS))
    return cache_error(LOG_IFERR(err));

  page_t *mp = nullptr;
  err = page_get(&cx.outer, txn->dbs[dbi].root, &mp, trunk_txnid);
  if (unlikely(err != MDBX_SUCCESS))
    return cache_error(LOG_IFERR(err));

  cx.outer.pg[cx.outer.top = 0] = mp;
  /* walking along the branches of the tree to the leaves */
  while (is_branch(mp)) {
    if (mp->txnid <= entry->last_confirmed_txnid) {
      /* Страница НЕ была обновлена после последней проверки,
       * проверяем остальные условия для раннего завершения поиска. */
      if (!entry->offset) {
        /* Ключа не было и он не мог быть добавлен без изменения страницы. */
      notfound_elevate_trunk:
        if (trunk_txnid > entry->trunk_txnid)
          goto not_found;
        for (intptr_t i = cx.outer.top - 1; i >= 0; --i) {
          trunk_txnid = cx.outer.pg[i]->txnid;
          if (trunk_txnid > entry->trunk_txnid)
            goto not_found;
        }
        trunk_txnid = txn->dbs[dbi].mod_txnid ? txn->dbs[dbi].mod_txnid : txn->front_txnid;
        cASSERT0(txn, trunk_txnid > entry->trunk_txnid);
        goto not_found;
      }
      if (mp->txnid < entry->trunk_txnid) {
        /* Попали на более старую страницу, такое может быть только при "схлопывании" b-tree,
         * когда целевой ключ был удален вместе с ведущими к нему branch-страницами. */
        goto notfound_elevate_trunk;
      }

      /* Искомый ключ больше или равен первому на этой странице и меньше следующего с родительской,
       * но может быть больше последнего ключа на этой странице. */
      if (!is_afterlast(&cx.outer, mp, &aligned.key)) {
        /* Следовательно, если ключ не-больше последнего на этой страницы,
         * то он не мог быть изменен или удалён. */
        trunk_txnid = mp->txnid;
        goto confirmed;
      }

      /* Но если искомый ключ больше последнего на этой страницы, то он мог быть удалён вместе со всеми
       * страницами расположенными дальше/после текущей и до следующего узла родительской страницы.
       * Следует продолжить поиск. */
    }

    trunk_txnid = mp->txnid;
    const size_t ki = tree_search_branch(&cx.outer, key);
    cx.outer.ki[cx.outer.top] = (indx_t)ki;
    err = page_get(&cx.outer, node_pgno(page_node(mp, ki)), &mp, trunk_txnid);
    if (unlikely(err != MDBX_SUCCESS))
      return cache_error(LOG_IFERR(err));

    err = cursor_push(&cx.outer, mp, 0);
    if (unlikely(err != MDBX_SUCCESS))
      return cache_error(LOG_IFERR(err));
  }

  if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(&cx.outer, mp))) {
    ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
    err = MDBX_CORRUPTED;
    return cache_error(LOG_IFERR(err));
  }

  if (mp->txnid <= entry->last_confirmed_txnid) {
    if (!entry->offset) {
      /* Ключа не было и он не мог быть добавлен без изменения страницы. */
      goto notfound_elevate_trunk;
    }
    if (mp->txnid < entry->trunk_txnid) {
      /* Попали на более старую страницу, такое может быть только при "схлопывании" b-tree,
       * когда целевой ключ был удален вместе с ведущими к нему branch-страницами. */
      tASSERT1(txn, !tree_search_foliage(&cx.outer, &aligned.key).exact);
      goto notfound_elevate_trunk;
    }

    /* Искомый ключ больше или равен первому на этой странице и меньше следующего с родительской,
     * но может быть больше последнего ключа на этой странице. Страница не была обновлена после
     * последней проверки наличия ключа. */
    if (!is_afterlast(&cx.outer, mp, &aligned.key)) {
      /* Следовательно, если ключ не-больше последнего на этой страницы,
       * то он не мог быть изменен или удалён. */
      trunk_txnid = mp->txnid;
      goto confirmed;
    }

    /* Но если искомый ключ больше последнего на этой страницы, то он мог быть удалён вместе со всеми
     * страницами расположенными дальше/после текущей и до следующего узла родительской страницы.
     *
     * В зоне branch-страниц дерева следовало-бы продолжить поиск, но в случае листовой страницы можно
     * быть уверенным что ключ отсутствует, так он больше последнего ключа на этой странице и меньше
     * первого на следующей, а таких ключей нет физически. */
    tASSERT1(txn, !tree_search_foliage(&cx.outer, &aligned.key).exact);
    goto notfound_elevate_trunk;
  }

  sfr_t sfr = tree_search_foliage(&cx.outer, &aligned.key);
  if (!sfr.exact) {
    cASSERT0(txn, !entry->offset || trunk_txnid > entry->trunk_txnid);
    if (sfr.node)
      /* Обновлять метку trunk_txnid можно только если искомый ключ должен был находится на текущей листовой странице,
       * но не в случае когда он болмьше последнего. Тогда trunk_txnid будет связан с родительской страницей и при
       * добавлении искомого ключа этом изменение можно будет отследить. Иначе trunk_txnid будет связан с текущей
       * листовой страницей, которая может не измениться при добавлении искомого ключа, так как он больше последнего
       * на этой странице и может быть добавлен в следующую страницу справа. Такое несоответствие может стать причиной
       * пропуска изменений и/или выдачи неверного результата. */
      trunk_txnid = mp->txnid;
    goto not_found;
  }

  trunk_txnid = mp->txnid;
  if (unlikely(node_flags(sfr.node) & N_DUP)) {
    /* TODO: It is possible to implement support for multivalues, but need to think through the usage scenarios. */
    err = MDBX_EMULTIVAL;
    return cache_error(LOG_IFERR(err));
  }

  err = node_read(&cx.outer, sfr.node, data, mp);
  if (unlikely(err != MDBX_SUCCESS))
    return cache_error(LOG_IFERR(err));

  if (trunk_txnid > committed_snapshot_txnid) {
    cASSERT0(txn, trunk_txnid > entry->last_confirmed_txnid && trunk_txnid > entry->trunk_txnid);
    return cache_result(MDBX_SUCCESS, MDBX_CACHE_DIRTY);
  }

  tASSERT0(txn, is_inside_dxb_and_commited(txn, data->iov_base));
  cASSERT0(txn, trunk_txnid <= committed_snapshot_txnid && trunk_txnid > entry->last_confirmed_txnid &&
                    trunk_txnid > entry->trunk_txnid);
  entry->offset = ptr_dist(data->iov_base, txn->env->dxb_mmap.base);
  entry->length = (uint32_t)data->iov_len;
  entry->trunk_txnid = trunk_txnid;
  entry->last_confirmed_txnid = committed_snapshot_txnid;
  return cache_result(MDBX_SUCCESS, MDBX_CACHE_REFRESHED);
}

/*----------------------------------------------------------------------------*/

__hot MDBX_cache_result_t mdbx_cache_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
                                         volatile MDBX_cache_entry_t *entry) {

  if (unlikely(!key || !data || !entry))
    return cache_error(LOG_IFERR(MDBX_EINVAL));

  MDBX_cache_entry_t local = *entry;
  while (true) {
    MDBX_cache_entry_t again;
    again.last_confirmed_txnid = safe64_read((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid);
    if (unlikely(again.last_confirmed_txnid > MAX_TXNID)) {
#if MDBX_DEBUG > 0
      return cache_fallback(txn, dbi, key, data, MDBX_CACHE_RACE);
#else
      atomic_yield();
      again.last_confirmed_txnid = safe64_read((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid);
      if (unlikely(again.last_confirmed_txnid > MAX_TXNID)) {
        atomic_yield();
        atomic_yield();
        again.last_confirmed_txnid = safe64_read((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid);
        if (unlikely(again.last_confirmed_txnid > MAX_TXNID)) {
          atomic_yield();
          atomic_yield();
          atomic_yield();
          again.last_confirmed_txnid = safe64_read((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid);
          if (unlikely(again.last_confirmed_txnid > MAX_TXNID))
            return cache_fallback(txn, dbi, key, data, MDBX_CACHE_RACE);
        }
      }
#endif /* MDBX_DEBUG */
    }

    again.trunk_txnid = entry->trunk_txnid;
    again.offset = entry->offset;
    again.length = entry->length;
    if (likely(local.last_confirmed_txnid == again.last_confirmed_txnid && local.trunk_txnid == again.trunk_txnid &&
               local.offset == again.offset && local.length == again.length &&
               local.last_confirmed_txnid == safe64_read((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid)))
      break;

    local = again;
    atomic_yield();
  }

  MDBX_cache_result_t result = cache_get(txn, dbi, key, data, &local);
  if (result.status > MDBX_CACHE_HIT) {
    cASSERT0(txn, local.last_confirmed_txnid < MAX_TXNID && local.trunk_txnid <= local.last_confirmed_txnid &&
                      local.trunk_txnid > 0);
    while (true) {
      const txnid_t snap = safe64_read((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid);
      if (snap >= local.last_confirmed_txnid) {
        result.status = MDBX_CACHE_RACE;
        break;
      }

      if (likely(safe64_reset_compare((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid, snap))) {
        entry->trunk_txnid = 0;
        osal_compiler_barrier();
        entry->offset = local.offset;
        entry->length = local.length;
        entry->trunk_txnid = local.trunk_txnid;
        safe64_write((mdbx_atomic_uint64_t *)&entry->last_confirmed_txnid, local.last_confirmed_txnid);
        break;
      }

      atomic_yield();
    }
  }
  return result;
}

__hot MDBX_cache_result_t mdbx_cache_get_SingleThreaded(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
                                                        MDBX_val *data, MDBX_cache_entry_t *entry) {
  if (unlikely(!key || !data || !entry))
    return cache_error(LOG_IFERR(MDBX_EINVAL));

  return cache_get(txn, dbi, key, data, entry);
}

#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API

LIBMDBX_API void mdbx_cache_init(MDBX_cache_entry_t *entry) { __inline_mdbx_cache_init(entry); }

#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */

static inline double key2double(const int64_t key) {
  union {
    uint64_t u;
    double f;
  } casting;

  casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000) : UINT64_C(0xffffFFFFffffFFFF) - key;
  return casting.f;
}

static inline uint64_t double2key(const double *const ptr) {
  STATIC_ASSERT(sizeof(double) == sizeof(int64_t));
  const int64_t i = *(const int64_t *)ptr;
  const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i : i + UINT64_C(0x8000000000000000);
  if (CHECKS1_ENABLED()) {
    const double f = key2double(u);
    ENSURE(memcmp(&f, ptr, sizeof(double)) == 0);
  }
  return u;
}

static inline float key2float(const int32_t key) {
  union {
    uint32_t u;
    float f;
  } casting;

  casting.u = (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key;
  return casting.f;
}

static inline uint32_t float2key(const float *const ptr) {
  STATIC_ASSERT(sizeof(float) == sizeof(int32_t));
  const int32_t i = *(const int32_t *)ptr;
  const uint32_t u = (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000);
  if (CHECKS1_ENABLED()) {
    const float f = key2float(u);
    ENSURE(memcmp(&f, ptr, sizeof(float)) == 0);
  }
  return u;
}

uint64_t mdbx_key_from_double(const double ieee754_64bit) { return double2key(&ieee754_64bit); }

uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) { return double2key(ieee754_64bit); }

uint32_t mdbx_key_from_float(const float ieee754_32bit) { return float2key(&ieee754_32bit); }

uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) { return float2key(ieee754_32bit); }

#define IEEE754_DOUBLE_MANTISSA_SIZE 52
#define IEEE754_DOUBLE_EXPONENTA_BIAS 0x3FF
#define IEEE754_DOUBLE_EXPONENTA_MAX 0x7FF
#define IEEE754_DOUBLE_IMPLICIT_LEAD UINT64_C(0x0010000000000000)
#define IEEE754_DOUBLE_MANTISSA_MASK UINT64_C(0x000FFFFFFFFFFFFF)
#define IEEE754_DOUBLE_MANTISSA_AMAX UINT64_C(0x001FFFFFFFFFFFFF)

static inline int clz64(uint64_t value) {
#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_clzl)
  if (sizeof(value) == sizeof(int))
    return __builtin_clz(value);
  if (sizeof(value) == sizeof(long))
    return __builtin_clzl(value);
#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_clzll)
  return __builtin_clzll(value);
#endif /* have(long long) && long long == uint64_t */
#endif /* GNU C */

#if defined(_MSC_VER)
  unsigned long index;
#if defined(_M_AMD64) || defined(_M_ARM64) || defined(_M_X64)
  _BitScanReverse64(&index, value);
  return 63 - index;
#else
  if (value > UINT32_MAX) {
    _BitScanReverse(&index, (uint32_t)(value >> 32));
    return 31 - index;
  }
  _BitScanReverse(&index, (uint32_t)value);
  return 63 - index;
#endif
#endif /* MSVC */

  value |= value >> 1;
  value |= value >> 2;
  value |= value >> 4;
  value |= value >> 8;
  value |= value >> 16;
  value |= value >> 32;
  static const uint8_t debruijn_clz64[64] = {63, 16, 62, 7,  15, 36, 61, 3,  6,  14, 22, 26, 35, 47, 60, 2,
                                             9,  5,  28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1,
                                             17, 8,  37, 4,  23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18,
                                             38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0};
  return debruijn_clz64[value * UINT64_C(0x03F79D71B4CB0A89) >> 58];
}

static inline uint64_t round_mantissa(const uint64_t u64, int shift) {
  ASSERT(shift < 0 && u64 > 0);
  shift = -shift;
  const unsigned half = 1 << (shift - 1);
  const unsigned lsb = 1 & (unsigned)(u64 >> shift);
  const unsigned tie2even = 1 ^ lsb;
  return (u64 + half - tie2even) >> shift;
}

uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) {
  const uint64_t bias = UINT64_C(0x8000000000000000);
  if (json_integer > 0) {
    const uint64_t u64 = json_integer;
    int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1);
    uint64_t mantissa = u64 << shift;
    if (unlikely(shift < 0)) {
      mantissa = round_mantissa(u64, shift);
      if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX)
        mantissa = round_mantissa(u64, --shift);
    }

    ASSERT(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
    const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
    ASSERT(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
    const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) + (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
#if !defined(_MSC_VER) || !MDBX_WITHOUT_MSVC_CRT /* Workaround for MSVC error LNK2019: unresolved external             \
                                             symbol __except1 referenced in function __ftol3_except */
    ASSERT(key == mdbx_key_from_double((double)json_integer));
#endif /* Workaround for MSVC */
    return key;
  }

  if (json_integer < 0) {
    const uint64_t u64 = -json_integer;
    int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1);
    uint64_t mantissa = u64 << shift;
    if (unlikely(shift < 0)) {
      mantissa = round_mantissa(u64, shift);
      if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX)
        mantissa = round_mantissa(u64, --shift);
    }

    ASSERT(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
    const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
    ASSERT(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
    const uint64_t key =
        bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
#if !defined(_MSC_VER) || !MDBX_WITHOUT_MSVC_CRT /* Workaround for MSVC error LNK2019: unresolved external             \
                                             symbol __except1 referenced in function __ftol3_except */
    ASSERT(key == mdbx_key_from_double((double)json_integer));
#endif /* Workaround for MSVC */
    return key;
  }

  return bias;
}

int64_t mdbx_jsonInteger_from_key(const MDBX_val v) {
  ASSERT(v.iov_len == 8);
  const uint64_t key = unaligned_peek_u64(2, v.iov_base);
  const uint64_t bias = UINT64_C(0x8000000000000000);
  const uint64_t covalent = (key > bias) ? key - bias : bias - key - 1;
  const int shift = IEEE754_DOUBLE_EXPONENTA_BIAS + 63 -
                    (IEEE754_DOUBLE_EXPONENTA_MAX & (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE));
  if (unlikely(shift < 1))
    return (key < bias) ? INT64_MIN : INT64_MAX;
  if (unlikely(shift > 63))
    return 0;

  const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK) << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) + bias;
  const int64_t absolute = unscaled >> shift;
  const int64_t value = (key < bias) ? -absolute : absolute;
  ASSERT(key == mdbx_key_from_jsonInteger(value) ||
         (mdbx_key_from_jsonInteger(value - 1) < key && key < mdbx_key_from_jsonInteger(value + 1)));
  return value;
}

double mdbx_double_from_key(const MDBX_val v) {
  ASSERT(v.iov_len == 8);
  return key2double(unaligned_peek_u64(2, v.iov_base));
}

float mdbx_float_from_key(const MDBX_val v) {
  ASSERT(v.iov_len == 4);
  return key2float(unaligned_peek_u32(2, v.iov_base));
}

int32_t mdbx_int32_from_key(const MDBX_val v) {
  ASSERT(v.iov_len == 4);
  return (int32_t)(unaligned_peek_u32(2, v.iov_base) - UINT32_C(0x80000000));
}

int64_t mdbx_int64_from_key(const MDBX_val v) {
  ASSERT(v.iov_len == 8);
  return (int64_t)(unaligned_peek_u64(2, v.iov_base) - UINT64_C(0x8000000000000000));
}

__cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) {
  if (volume <= 1024 * 1024 * 4ul)
    return MDBX_RESULT_TRUE;

  intptr_t pagesize, total_ram_pages;
  int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr);
  if (unlikely(err != MDBX_SUCCESS))
    return LOG_IFERR(err);

  const int log2page = log2n_powerof2(pagesize);
  const intptr_t volume_pages = (volume + pagesize - 1) >> log2page;
  const intptr_t redundancy_pages = (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page)
                                                     : (intptr_t)(redundancy + pagesize - 1) >> log2page;
  if (volume_pages >= total_ram_pages || volume_pages + redundancy_pages >= total_ram_pages)
    return MDBX_RESULT_FALSE;

  intptr_t avail_ram_pages;
  err = mdbx_get_sysraminfo(nullptr, nullptr, &avail_ram_pages);
  if (unlikely(err != MDBX_SUCCESS))
    return LOG_IFERR(err);

  return (volume_pages + redundancy_pages >= avail_ram_pages) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
}

int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, uint64_t increment) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = dbi_check(txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(txn->dbi_state[dbi] & DBI_STALE)) {
    rc = tbl_refresh_absent2baddbi(txn, dbi);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  tree_t *dbs = &txn->dbs[dbi];
  if (likely(result))
    *result = dbs->sequence;

  if (likely(increment > 0)) {
    if (unlikely(dbi == FREE_DBI || (txn->flags & txn_ro_both) != 0))
      return MDBX_EACCESS;

    uint64_t new = dbs->sequence + increment;
    if (unlikely(new < increment))
      return MDBX_RESULT_TRUE;

    cASSERT0(txn, new > dbs->sequence);
    if ((txn->dbi_state[dbi] & DBI_DIRTY) == 0) {
      txn->flags |= MDBX_TXN_DIRTY;
      txn->dbi_state[dbi] |= DBI_DIRTY;
      if (unlikely(dbi == MAIN_DBI) && txn->dbs[MAIN_DBI].root != P_INVALID) {
        /* LY: Временная подпорка для coherency_check(), которую в перспективе
         * следует заменить вместе с переделкой установки mod_txnid.
         *
         * Суть проблемы:
         *  - coherency_check() в качестве одного из критериев "когерентности"
         *    проверяет условие meta.maindb.mod_txnid == maindb.root->txnid;
         *  - при обновлении maindb.sequence высталяется DBI_DIRTY, что приведет
         *    к обновлению meta.maindb.mod_txnid = current_txnid;
         *  - однако, если в само дерево maindb обновление не вносились и оно
         *    не пустое, то корневая страницы останется с прежним txnid и из-за
         *    этого ложно сработает coherency_check().
         *
         * Временное (текущее) решение: Принудительно обновляем корневую
         * страницу в описанной выше ситуации. Это устраняет проблему, но и
         * не создает рисков регресса.
         *
         * FIXME: Итоговое решение, которое предстоит реализовать:
         *  - изменить семантику установки/обновления mod_txnid, привязав его
         *    строго к изменению b-tree, но не атрибутов;
         *  - обновлять mod_txnid при фиксации вложенных транзакций;
         *  - для dbi-хендлов пользовательских table (видимо) можно оставить
         *    DBI_DIRTY в качестве признака необходимости обновления записи
         *    table в MainDB, при этом взводить DBI_DIRTY вместе с обновлением
         *    mod_txnid, в том числе при обновлении sequence.
         *  - для MAIN_DBI при обновлении sequence не следует взводить DBI_DIRTY
         *    и/или обновлять mod_txnid, а только взводить MDBX_TXN_DIRTY.
         *  - альтернативно, можно перераспределить флажки-признаки dbi_state,
         *    чтобы различать состояние dirty-tree и dirty-attributes. */
        cursor_couple_t cx;
        rc = cursor_init(&cx.outer, txn, MAIN_DBI);
        if (unlikely(rc != MDBX_SUCCESS))
          return LOG_IFERR(rc);
        rc = tree_search(&cx.outer, nullptr, Z_MODIFY | Z_ROOTONLY);
        if (unlikely(rc != MDBX_SUCCESS))
          return LOG_IFERR(rc);
      }
    }
    dbs->sequence = new;
  }

  return MDBX_SUCCESS;
}

int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) {
  tASSERT0(txn, txn->signature == txn_signature);
  tASSERT0(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi));
  tASSERT0(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0);
  return txn->env->kvs[dbi].clc.k.cmp(a, b);
}

int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) {
  tASSERT0(txn, txn->signature == txn_signature);
  tASSERT0(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi));
  tASSERT0(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID));
  return txn->env->kvs[dbi].clc.v.cmp(a, b);
}

__cold MDBX_cmp_func mdbx_get_keycmp(MDBX_db_flags_t flags) { return builtin_keycmp(flags); }

__cold MDBX_cmp_func mdbx_get_datacmp(MDBX_db_flags_t flags) { return builtin_datacmp(flags); }

/*----------------------------------------------------------------------------*/

__cold const char *mdbx_liberr2str(int errnum) {
  /* Table of descriptions for MDBX errors */
  static const char *const tbl[] = {
      "MDBX_KEYEXIST: Key/data pair already exists",
      "MDBX_NOTFOUND: No matching key/data pair found",
      "MDBX_PAGE_NOTFOUND: Requested page not found",
      "MDBX_CORRUPTED: Database is corrupted",
      "MDBX_PANIC: Environment had fatal error",
      "MDBX_VERSION_MISMATCH: DB version mismatch libmdbx",
      "MDBX_INVALID: File is not an MDBX file",
      "MDBX_MAP_FULL: Environment mapsize limit reached",
      "MDBX_DBS_FULL: Too many DBI-handles (maxdbs reached)",
      "MDBX_READERS_FULL: Too many readers (maxreaders reached)",
      nullptr /* MDBX_TLS_FULL (-30789): unused in MDBX */,
      "MDBX_TXN_FULL: Transaction has too many dirty pages,"
      " i.e transaction is too big",
      "MDBX_CURSOR_FULL: Cursor stack limit reached - this usually indicates"
      " corruption, i.e branch-pages loop",
      "MDBX_PAGE_FULL: Internal error - Page has no more space",
      "MDBX_UNABLE_EXTEND_MAPSIZE: Database engine was unable to extend"
      " mapping, e.g. since address space is unavailable or busy,"
      " or Operation system not supported such operations",
      "MDBX_INCOMPATIBLE: Environment or database is not compatible"
      " with the requested operation or the specified flags",
      "MDBX_BAD_RSLOT: Reader locktable slot was unexpectly reused or cleared by an enemy thread",
      "MDBX_BAD_TXN: Transaction is not valid for requested operation,"
      " e.g. had errored and be must aborted, has a child, or is invalid",
      "MDBX_BAD_VALSIZE: Invalid size or alignment of key or data"
      " for target database, either invalid table name",
      "MDBX_BAD_DBI: The specified DBI-handle is invalid"
      " or changed by another thread/transaction",
      "MDBX_PROBLEM: Unexpected internal error, transaction should be aborted",
      "MDBX_BUSY: Another write transaction is running,"
      " or environment is already used while opening with MDBX_EXCLUSIVE flag",
  };

  if (errnum >= MDBX_KEYEXIST && errnum <= MDBX_BUSY) {
    int i = errnum - MDBX_KEYEXIST;
    return tbl[i];
  }

  switch (errnum) {
  case MDBX_SUCCESS:
    return "MDBX_SUCCESS: Successful";
  case MDBX_EMULTIVAL:
    return "MDBX_EMULTIVAL: The specified key has"
           " more than one associated value";
  case MDBX_EBADSIGN:
    return "MDBX_EBADSIGN: Wrong signature of a runtime object(s),"
           " e.g. memory corruption or double-free";
  case MDBX_WANNA_RECOVERY:
    return "MDBX_WANNA_RECOVERY: Database should be recovered,"
           " but this could NOT be done automatically for now"
           " since it opened in read-only mode";
  case MDBX_EKEYMISMATCH:
    return "MDBX_EKEYMISMATCH: The given key value is mismatched to the"
           " current cursor position";
  case MDBX_TOO_LARGE:
    return "MDBX_TOO_LARGE: Database is too large for current system,"
           " e.g. could NOT be mapped into RAM";
  case MDBX_THREAD_MISMATCH:
    return "MDBX_THREAD_MISMATCH: A thread has attempted to use a not"
           " owned object, e.g. a transaction that started by another thread";
  case MDBX_TXN_OVERLAPPING:
    return "MDBX_TXN_OVERLAPPING: Overlapping read and write transactions for"
           " the current thread";
  case MDBX_DUPLICATED_CLK:
    return "MDBX_DUPLICATED_CLK: Alternative/Duplicate LCK-file is exists,"
           " please keep one and remove unused other";
  case MDBX_DANGLING_DBI:
    return "MDBX_DANGLING_DBI: Some cursors and/or other resources should be"
           " closed before table or corresponding DBI-handle could be (re)used";
  case MDBX_OUSTED:
    return "MDBX_OUSTED: The parked read transaction was outed for the sake"
           " of recycling old MVCC snapshots";
  case MDBX_MVCC_RETARDED:
    return "MDBX_MVCC_RETARDED: MVCC snapshot used by parked transaction was bygone";
  case MDBX_LAGGARD_READER:
    return "An operation cannot continue because a lagging reader is interfering with the"
           " reclaiming of GC and old MVCC-snapshots";
  default:
    return nullptr;
  }
}

__cold const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen) {
  const char *msg = mdbx_liberr2str(errnum);
  if (!msg && buflen > 0 && buflen < INT_MAX) {
#if defined(_WIN32) || defined(_WIN64)
    DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum,
                                MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr);
    while (size && buf[size - 1] <= ' ')
      --size;
    buf[size] = 0;
    return size ? buf : "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed";
#elif defined(_GNU_SOURCE) && defined(__GLIBC__)
    /* GNU-specific */
    if (errnum > 0)
      msg = strerror_r(errnum, buf, buflen);
#elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600)
    /* XSI-compliant */
    if (errnum > 0 && strerror_r(errnum, buf, buflen) == 0)
      msg = buf;
#else
    if (errnum > 0) {
      msg = strerror(errnum);
      if (msg) {
        strncpy(buf, msg, buflen);
        msg = buf;
      }
    }
#endif
    if (!msg) {
      (void)snprintf(buf, buflen, "error %d", errnum);
      msg = buf;
    }
    buf[buflen - 1] = '\0';
  }
  return msg;
}

__cold const char *mdbx_strerror(int errnum) {
#if defined(_WIN32) || defined(_WIN64)
  static char buf[1024];
  return mdbx_strerror_r(errnum, buf, sizeof(buf));
#else
  const char *msg = mdbx_liberr2str(errnum);
  if (!msg) {
    if (errnum > 0)
      msg = strerror(errnum);
    if (!msg) {
      static char buf[32];
      (void)snprintf(buf, sizeof(buf) - 1, "error %d", errnum);
      msg = buf;
    }
  }
  return msg;
#endif
}

#if defined(_WIN32) || defined(_WIN64) /* Bit of madness for Windows */
const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen) {
  const char *msg = mdbx_liberr2str(errnum);
  if (!msg && buflen > 0 && buflen < INT_MAX) {
    DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum,
                                MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr);
    while (size && buf[size - 1] <= ' ')
      --size;
    buf[size] = 0;
    if (!size)
      msg = "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed";
    else if (!CharToOemBuffA(buf, buf, size))
      msg = "CharToOemBuffA() failed";
    else
      msg = buf;
  }
  return msg;
}

const char *mdbx_strerror_ANSI2OEM(int errnum) {
  static char buf[1024];
  return mdbx_strerror_r_ANSI2OEM(errnum, buf, sizeof(buf));
}
#endif /* Bit of madness for Windows */

static pgno_t env_max_pgno(const MDBX_env *env) {
  return env->ps ? bytes2pgno(env, env->geo_in_bytes.upper ? env->geo_in_bytes.upper : MAX_MAPSIZE) : PAGELIST_LIMIT;
}

__cold pgno_t default_dp_limit(const MDBX_env *env) {
  /* auto-setup dp_limit by "The42" ;-) */
  intptr_t total_ram_pages, avail_ram_pages;
  int err = mdbx_get_sysraminfo(nullptr, &total_ram_pages, &avail_ram_pages);
  pgno_t dp_limit = 1024;
  if (unlikely(err != MDBX_SUCCESS))
    ERROR("mdbx_get_sysraminfo(), rc %d", err);
  else {
    size_t estimate = (size_t)(total_ram_pages + avail_ram_pages) / 42;
    if (env->ps) {
      if (env->ps > globals.sys_pagesize)
        estimate /= env->ps / globals.sys_pagesize;
      else if (env->ps < globals.sys_pagesize)
        estimate *= globals.sys_pagesize / env->ps;
    }
    dp_limit = (pgno_t)estimate;
  }

  dp_limit = (dp_limit < PAGELIST_LIMIT) ? dp_limit : PAGELIST_LIMIT;
  const pgno_t max_pgno = env_max_pgno(env);
  if (dp_limit > max_pgno - NUM_METAS)
    dp_limit = max_pgno - NUM_METAS;
  dp_limit = (dp_limit > CURSOR_STACK_SIZE * 4) ? dp_limit : CURSOR_STACK_SIZE * 4;
  return dp_limit;
}

__cold static pgno_t default_rp_augment_limit(const MDBX_env *env) {
  const size_t timeframe = /* 16 секунд */ 16 << 16;
  const size_t remain_1sec =
      (env->options.gc_time_limit < timeframe) ? timeframe - (size_t)env->options.gc_time_limit : 0;
  const size_t minimum = (env->maxgc_large1page * 2 > MDBX_PNL_INITIAL) ? env->maxgc_large1page * 2 : MDBX_PNL_INITIAL;
  const size_t one_third = env->geo_in_bytes.now / 3 >> env->ps2ln;
  const size_t augment_limit =
      (one_third > minimum) ? minimum + (one_third - minimum) / timeframe * remain_1sec : minimum;
  eASSERT0(env, augment_limit < PAGELIST_LIMIT);
  return pnl_bytes2size(pnl_size2bytes(augment_limit));
}

static bool default_prefault_write(const MDBX_env *env) {
  return !MDBX_MMAP_INCOHERENT_FILE_WRITE && !env->incore &&
         (env->flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == MDBX_WRITEMAP;
}

static bool default_prefer_waf_insteadof_balance(const MDBX_env *env) {
  (void)env;
  return true;
}

static uint16_t default_subpage_limit(const MDBX_env *env) {
  (void)env;
  return 65535 /* 100% */;
}

static uint16_t default_subpage_room_threshold(const MDBX_env *env) {
  (void)env;
  return 0 /* 0% */;
}

static uint16_t default_subpage_reserve_prereq(const MDBX_env *env) {
  (void)env;
  return 27525 /* 42% */;
}

static uint16_t default_subpage_reserve_limit(const MDBX_env *env) {
  (void)env;
  return 2753 /* 4.2% */;
}

static uint16_t default_merge_threshold_dot16(const MDBX_env *env) {
  (void)env;
  return 65536 / 3 /* 33% */;
}

static pgno_t default_dp_reserve_limit(const MDBX_env *env) {
  (void)env;
  return MDBX_PNL_INITIAL;
}

static pgno_t default_dp_initial(const MDBX_env *env) {
  (void)env;
  return MDBX_PNL_INITIAL;
}

static uint8_t default_spill_max_denominator(const MDBX_env *env) {
  (void)env;
  return 8;
}

static uint8_t default_spill_min_denominator(const MDBX_env *env) {
  (void)env;
  return 8;
}

static uint8_t default_spill_parent4child_denominator(const MDBX_env *env) {
  (void)env;
  return 0;
}

static uint8_t default_dp_loose_limit(const MDBX_env *env) {
  (void)env;
  return 64;
}

void env_options_init(MDBX_env *env) {
  env->options.rp_augment_limit = default_rp_augment_limit(env);
  env->options.dp_reserve_limit = default_dp_reserve_limit(env);
  env->options.dp_initial = default_dp_initial(env);
  env->options.dp_limit = (/* just to avoid extra syscalls and because dp_limit will be adjusted later */ true)
                              ? env->options.dp_initial * 42
                              : default_dp_limit(env);
  env->options.spill_max_denominator = default_spill_max_denominator(env);
  env->options.spill_min_denominator = default_spill_min_denominator(env);
  env->options.spill_parent4child_denominator = default_spill_parent4child_denominator(env);
  env->options.dp_loose_limit = default_dp_loose_limit(env);
  env->options.merge_threshold_dot16 = default_merge_threshold_dot16(env);
  if (default_prefer_waf_insteadof_balance(env))
    env->options.prefer_waf_insteadof_balance = true;

#if !(defined(_WIN32) || defined(_WIN64))
  env->options.writethrough_threshold =
#if defined(__linux__) || defined(__gnu_linux__)
      globals.running_on_WSL1 ? MAX_PAGENO :
#endif /* Linux */
                              MDBX_WRITETHROUGH_THRESHOLD_DEFAULT;
#endif /* Windows */

  env->options.subpage.limit = default_subpage_limit(env);
  env->options.subpage.room_threshold = default_subpage_room_threshold(env);
  env->options.subpage.reserve_prereq = default_subpage_reserve_prereq(env);
  env->options.subpage.reserve_limit = default_subpage_reserve_limit(env);
}

void env_options_adjust_dp_limit(MDBX_env *env) {
  if (!env->options.flags.non_auto.dp_limit)
    env->options.dp_limit = default_dp_limit(env);
  else {
    const pgno_t max_pgno = env_max_pgno(env);
    if (env->options.dp_limit > max_pgno - NUM_METAS)
      env->options.dp_limit = max_pgno - NUM_METAS;
    if (env->options.dp_limit < CURSOR_STACK_SIZE * 4)
      env->options.dp_limit = CURSOR_STACK_SIZE * 4;
  }
#ifdef MDBX_DEBUG_DPL_LIMIT
  env->options.dp_limit = MDBX_DEBUG_DPL_LIMIT;
#endif /* MDBX_DEBUG_DPL_LIMIT */
  if (env->options.dp_initial > env->options.dp_limit && env->options.dp_initial > default_dp_initial(env))
    env->options.dp_initial = env->options.dp_limit;
  env->options.need_dp_limit_adjust = false;
}

void env_options_adjust_defaults(MDBX_env *env) {
  if (!env->options.flags.non_auto.rp_augment_limit)
    env->options.rp_augment_limit = default_rp_augment_limit(env);
  if (!env->options.flags.non_auto.prefault_write)
    env->options.prefault_write = default_prefault_write(env);

  env->options.need_dp_limit_adjust = true;
  if (!env->txn)
    env_options_adjust_dp_limit(env);

  const size_t basis = env->geo_in_bytes.now;
  /* TODO: use options? */
  const unsigned factor = 9;
  size_t threshold = (basis < ((size_t)65536 << factor))  ? 65536        /* minimal threshold */
                     : (basis > (MEGABYTE * 4 << factor)) ? MEGABYTE * 4 /* maximal threshold */
                                                          : basis >> factor;
  threshold =
      (threshold < env->geo_in_bytes.shrink || !env->geo_in_bytes.shrink) ? threshold : env->geo_in_bytes.shrink;
  env->madv_threshold = bytes_ceil2os_pgno(env, threshold);
}

//------------------------------------------------------------------------------

__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, uint64_t value) {
  int err = check_env(env, false);
  if (unlikely(err != MDBX_SUCCESS))
    return LOG_IFERR(err);

  const bool lock_needed = ((env->flags & ENV_ACTIVE) && env->basal_txn && !env_owned_wrtxn(env));
  bool should_unlock = false;
  switch (option) {
  case MDBX_opt_sync_bytes:
    if (value == /* default */ UINT64_MAX)
      value = MAX_WRITE;
    if (unlikely(env->flags & MDBX_RDONLY))
      return LOG_IFERR(MDBX_EACCESS);
    if (unlikely(!(env->flags & ENV_ACTIVE)))
      return LOG_IFERR(MDBX_EPERM);
    if (unlikely(value > SIZE_MAX - 65536))
      return LOG_IFERR(MDBX_EINVAL);
    value = bytes2pgno(env, (size_t)value + env->ps - 1);
    if ((uint32_t)value != atomic_load32(&env->lck->autosync_threshold, mo_AcquireRelease) &&
        atomic_store32(&env->lck->autosync_threshold, (uint32_t)value, mo_Relaxed)
        /* Дергаем sync(force=off) только если задано новое не-нулевое значение
         * и мы вне транзакции */
        && lock_needed) {
      err = env_sync(env, false, false);
      if (err == /* нечего сбрасывать на диск */ MDBX_RESULT_TRUE)
        err = MDBX_SUCCESS;
    }
    break;

  case MDBX_opt_sync_period:
    if (value == /* default */ UINT64_MAX)
      value = 2780315 /* 42.42424 секунды */;
    if (unlikely(env->flags & MDBX_RDONLY))
      return LOG_IFERR(MDBX_EACCESS);
    if (unlikely(!(env->flags & ENV_ACTIVE)))
      return LOG_IFERR(MDBX_EPERM);
    if (unlikely(value > UINT32_MAX))
      return LOG_IFERR(MDBX_EINVAL);
    value = osal_16dot16_to_monotime((uint32_t)value);
    if (value != atomic_load64(&env->lck->autosync_period, mo_AcquireRelease) &&
        atomic_store64(&env->lck->autosync_period, value, mo_Relaxed)
        /* Дергаем sync(force=off) только если задано новое не-нулевое значение
         * и мы вне транзакции */
        && lock_needed) {
      err = env_sync(env, false, false);
      if (err == /* нечего сбрасывать на диск */ MDBX_RESULT_TRUE)
        err = MDBX_SUCCESS;
    }
    break;

  case MDBX_opt_max_db:
    if (value == /* default */ UINT64_MAX)
      value = 42;
    if (unlikely(value > MDBX_MAX_DBI))
      return LOG_IFERR(MDBX_EINVAL);
    if (unlikely(env->dxb_mmap.base))
      return LOG_IFERR(MDBX_EPERM);
    env->max_dbi = (unsigned)value + CORE_DBS;
    break;

  case MDBX_opt_max_readers:
    if (value == /* default */ UINT64_MAX)
      value = MDBX_READERS_LIMIT;
    if (unlikely(value < 1 || value > MDBX_READERS_LIMIT))
      return LOG_IFERR(MDBX_EINVAL);
    if (unlikely(env->dxb_mmap.base))
      return LOG_IFERR(MDBX_EPERM);
    env->max_readers = (unsigned)value;
    break;

  case MDBX_opt_dp_reserve_limit:
    if (value == /* default */ UINT64_MAX)
      value = default_dp_reserve_limit(env);
    if (unlikely(value > INT_MAX))
      return LOG_IFERR(MDBX_EINVAL);
    if (env->options.dp_reserve_limit != (unsigned)value) {
      if (lock_needed) {
        err = lck_txn_lock(env, false);
        if (unlikely(err != MDBX_SUCCESS))
          return LOG_IFERR(err);
        should_unlock = true;
      }
      env->options.dp_reserve_limit = (unsigned)value;
      while (env->shadow_reserve_len > env->options.dp_reserve_limit) {
        eASSERT0(env, env->shadow_reserve != nullptr);
        page_t *dp = env->shadow_reserve;
        MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->ps);
        VALGRIND_MAKE_MEM_DEFINED(&page_next(dp), sizeof(page_t *));
        env->shadow_reserve = page_next(dp);
        void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t));
        osal_free(ptr);
        env->shadow_reserve_len -= 1;
      }
    }
    break;

  case MDBX_opt_rp_augment_limit:
    if (value == /* default */ UINT64_MAX) {
      env->options.flags.non_auto.rp_augment_limit = 0;
      env->options.rp_augment_limit = default_rp_augment_limit(env);
    } else if (unlikely(value > PAGELIST_LIMIT))
      return LOG_IFERR(MDBX_EINVAL);
    else {
      env->options.flags.non_auto.rp_augment_limit = 1;
      env->options.rp_augment_limit = (unsigned)value;
    }
    break;

  case MDBX_opt_gc_time_limit:
    if (value == /* default */ UINT64_MAX)
      value = 0;
    if (unlikely(value > UINT32_MAX))
      return LOG_IFERR(MDBX_EINVAL);
    if (unlikely(env->flags & MDBX_RDONLY))
      return LOG_IFERR(MDBX_EACCESS);
    value = osal_16dot16_to_monotime((uint32_t)value);
    if (value != env->options.gc_time_limit) {
      if (env->txn && lock_needed)
        return LOG_IFERR(MDBX_EPERM);
      env->options.gc_time_limit = value;
      if (!env->options.flags.non_auto.rp_augment_limit)
        env->options.rp_augment_limit = default_rp_augment_limit(env);
    }
    break;

  case MDBX_opt_txn_dp_limit:
  case MDBX_opt_txn_dp_initial:
    if (value != /* default */ UINT64_MAX && unlikely(value > PAGELIST_LIMIT || value < CURSOR_STACK_SIZE * 4))
      return LOG_IFERR(MDBX_EINVAL);
    if (unlikely(env->flags & MDBX_RDONLY))
      return LOG_IFERR(MDBX_EACCESS);
    if (lock_needed) {
      err = lck_txn_lock(env, false);
      if (unlikely(err != MDBX_SUCCESS))
        return LOG_IFERR(err);
      should_unlock = true;
    }
    if (env->txn)
      err = MDBX_EPERM /* unable change during transaction */;
    else {
      const pgno_t max_pgno = env_max_pgno(env);
      if (option == MDBX_opt_txn_dp_initial) {
        if (value == /* default */ UINT64_MAX)
          env->options.dp_initial = default_dp_initial(env);
        else {
          env->options.dp_initial = (pgno_t)value;
          if (env->options.dp_initial > max_pgno)
            env->options.dp_initial = (max_pgno > CURSOR_STACK_SIZE * 4) ? max_pgno : CURSOR_STACK_SIZE * 4;
        }
      }
      if (option == MDBX_opt_txn_dp_limit) {
        if (value == /* default */ UINT64_MAX) {
          env->options.flags.non_auto.dp_limit = 0;
        } else {
          env->options.flags.non_auto.dp_limit = 1;
          env->options.dp_limit = (pgno_t)value;
        }
        env_options_adjust_dp_limit(env);
      }
    }
    break;

  case MDBX_opt_spill_max_denominator:
    if (value == /* default */ UINT64_MAX)
      value = default_spill_max_denominator(env);
    if (unlikely(value > 255))
      return LOG_IFERR(MDBX_EINVAL);
    env->options.spill_max_denominator = (uint8_t)value;
    break;
  case MDBX_opt_spill_min_denominator:
    if (value == /* default */ UINT64_MAX)
      value = default_spill_min_denominator(env);
    if (unlikely(value > 255))
      return LOG_IFERR(MDBX_EINVAL);
    env->options.spill_min_denominator = (uint8_t)value;
    break;
  case MDBX_opt_spill_parent4child_denominator:
    if (value == /* default */ UINT64_MAX)
      value = default_spill_parent4child_denominator(env);
    if (unlikely(value > 255))
      return LOG_IFERR(MDBX_EINVAL);
    env->options.spill_parent4child_denominator = (uint8_t)value;
    break;

  case MDBX_opt_loose_limit:
    if (value == /* default */ UINT64_MAX)
      value = default_dp_loose_limit(env);
    if (unlikely(value > 255))
      return LOG_IFERR(MDBX_EINVAL);
    env->options.dp_loose_limit = (uint8_t)value;
    break;

  case MDBX_opt_merge_threshold:
    if (value == /* default */ UINT64_MAX)
      value = default_merge_threshold_dot16(env);
    if (unlikely(value < 8192 || value > 32768))
      return LOG_IFERR(MDBX_EINVAL);
    env->options.merge_threshold_dot16 = (unsigned)value;
    recalculate_merge_thresholds(env);
    break;

  case MDBX_opt_writethrough_threshold:
#if defined(_WIN32) || defined(_WIN64)
    /* позволяем "установить" значение по-умолчанию и совпадающее
     * с поведением соответствующим текущей установке MDBX_NOMETASYNC */
    if (value != /* default */ UINT64_MAX && value != ((env->flags & MDBX_NOMETASYNC) ? 0 : UINT_MAX))
      err = MDBX_EINVAL;
#else
    if (value == /* default */ UINT64_MAX)
      value = MDBX_WRITETHROUGH_THRESHOLD_DEFAULT;
    if (value != (unsigned)value)
      err = MDBX_EINVAL;
    else
      env->options.writethrough_threshold = (unsigned)value;
#endif
    break;

  case MDBX_opt_prefault_write_enable:
    if (value == /* default */ UINT64_MAX) {
      env->options.prefault_write = default_prefault_write(env);
      env->options.flags.non_auto.prefault_write = false;
    } else if (value > 1)
      err = MDBX_EINVAL;
    else {
      env->options.prefault_write = value != 0;
      env->options.flags.non_auto.prefault_write = true;
    }
    break;

  case MDBX_opt_prefer_waf_insteadof_balance:
    if (value == /* default */ UINT64_MAX)
      env->options.prefer_waf_insteadof_balance = default_prefer_waf_insteadof_balance(env);
    else if (value > 1)
      err = MDBX_EINVAL;
    else
      env->options.prefer_waf_insteadof_balance = value != 0;
    break;

  case MDBX_opt_subpage_limit:
    if (value == /* default */ UINT64_MAX) {
      env->options.subpage.limit = default_subpage_limit(env);
      recalculate_subpage_thresholds(env);
    } else if (value > 65535)
      err = MDBX_EINVAL;
    else {
      env->options.subpage.limit = (uint16_t)value;
      recalculate_subpage_thresholds(env);
    }
    break;

  case MDBX_opt_subpage_room_threshold:
    if (value == /* default */ UINT64_MAX) {
      env->options.subpage.room_threshold = default_subpage_room_threshold(env);
      recalculate_subpage_thresholds(env);
    } else if (value > 65535)
      err = MDBX_EINVAL;
    else {
      env->options.subpage.room_threshold = (uint16_t)value;
      recalculate_subpage_thresholds(env);
    }
    break;

  case MDBX_opt_subpage_reserve_prereq:
    if (value == /* default */ UINT64_MAX) {
      env->options.subpage.reserve_prereq = default_subpage_reserve_prereq(env);
      recalculate_subpage_thresholds(env);
    } else if (value > 65535)
      err = MDBX_EINVAL;
    else {
      env->options.subpage.reserve_prereq = (uint16_t)value;
      recalculate_subpage_thresholds(env);
    }
    break;

  case MDBX_opt_subpage_reserve_limit:
    if (value == /* default */ UINT64_MAX) {
      env->options.subpage.reserve_limit = default_subpage_reserve_limit(env);
      recalculate_subpage_thresholds(env);
    } else if (value > 65535)
      err = MDBX_EINVAL;
    else {
      env->options.subpage.reserve_limit = (uint16_t)value;
      recalculate_subpage_thresholds(env);
    }
    break;

  case MDBX_opt_split_reserve:
    if (value == /* default */ UINT64_MAX)
      env->options.split_reserve_dot16 = 0;
    else if (value > 32768)
      err = MDBX_EINVAL;
    else
      env->options.split_reserve_dot16 = (uint16_t)value;
    break;

  default:
    return LOG_IFERR(MDBX_EINVAL);
  }

  if (should_unlock)
    lck_txn_unlock(env);
  return LOG_IFERR(err);
}

__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, uint64_t *pvalue) {
  int err = check_env(env, false);
  if (unlikely(err != MDBX_SUCCESS))
    return LOG_IFERR(err);
  if (unlikely(!pvalue))
    return LOG_IFERR(MDBX_EINVAL);

  switch (option) {
  case MDBX_opt_sync_bytes:
    if (unlikely(!(env->flags & ENV_ACTIVE)))
      return LOG_IFERR(MDBX_EPERM);
    *pvalue = pgno2bytes(env, atomic_load32(&env->lck->autosync_threshold, mo_Relaxed));
    break;

  case MDBX_opt_sync_period:
    if (unlikely(!(env->flags & ENV_ACTIVE)))
      return LOG_IFERR(MDBX_EPERM);
    *pvalue = osal_monotime_to_16dot16(atomic_load64(&env->lck->autosync_period, mo_Relaxed));
    break;

  case MDBX_opt_max_db:
    *pvalue = env->max_dbi - CORE_DBS;
    break;

  case MDBX_opt_max_readers:
    *pvalue = env->max_readers;
    break;

  case MDBX_opt_dp_reserve_limit:
    *pvalue = env->options.dp_reserve_limit;
    break;

  case MDBX_opt_rp_augment_limit:
    *pvalue = env->options.rp_augment_limit;
    break;

  case MDBX_opt_gc_time_limit:
    *pvalue = osal_monotime_to_16dot16(env->options.gc_time_limit);
    break;

  case MDBX_opt_txn_dp_limit:
    *pvalue = env->options.dp_limit;
    break;
  case MDBX_opt_txn_dp_initial:
    *pvalue = env->options.dp_initial;
    break;

  case MDBX_opt_spill_max_denominator:
    *pvalue = env->options.spill_max_denominator;
    break;
  case MDBX_opt_spill_min_denominator:
    *pvalue = env->options.spill_min_denominator;
    break;
  case MDBX_opt_spill_parent4child_denominator:
    *pvalue = env->options.spill_parent4child_denominator;
    break;

  case MDBX_opt_loose_limit:
    *pvalue = env->options.dp_loose_limit;
    break;

  case MDBX_opt_merge_threshold:
    *pvalue = env->options.merge_threshold_dot16;
    break;

  case MDBX_opt_writethrough_threshold:
#if defined(_WIN32) || defined(_WIN64)
    *pvalue = (env->flags & MDBX_NOMETASYNC) ? 0 : INT_MAX;
#else
    *pvalue = env->options.writethrough_threshold;
#endif
    break;

  case MDBX_opt_prefault_write_enable:
    *pvalue = env->options.prefault_write;
    break;

  case MDBX_opt_prefer_waf_insteadof_balance:
    *pvalue = env->options.prefer_waf_insteadof_balance;
    break;

  case MDBX_opt_subpage_limit:
    *pvalue = env->options.subpage.limit;
    break;

  case MDBX_opt_subpage_room_threshold:
    *pvalue = env->options.subpage.room_threshold;
    break;

  case MDBX_opt_subpage_reserve_prereq:
    *pvalue = env->options.subpage.reserve_prereq;
    break;

  case MDBX_opt_subpage_reserve_limit:
    *pvalue = env->options.subpage.reserve_limit;
    break;

  case MDBX_opt_split_reserve:
    *pvalue = env->options.split_reserve_dot16;
    break;

  default:
    return LOG_IFERR(MDBX_EINVAL);
  }

  return MDBX_SUCCESS;
}

typedef struct diff_result {
  ptrdiff_t diff;
  intptr_t level;
  ptrdiff_t root_nkeys;
} diff_t;

/* calculates: r = x - y */
__hot static int cursor_diff(const MDBX_cursor *const __restrict x, const MDBX_cursor *const __restrict y,
                             diff_t *const __restrict r) {
  r->diff = 0;
  r->level = 0;
  r->root_nkeys = 0;

  int rc = check_txn(x->txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  if (unlikely(x->txn != y->txn))
    return MDBX_BAD_TXN;

  if (unlikely(y->dbi_state != x->dbi_state))
    return MDBX_EINVAL;

  const intptr_t depth = (x->top < y->top) ? x->top : y->top;
  if (unlikely(depth < 0))
    return MDBX_ENODATA;

  r->root_nkeys = page_numkeys(x->pg[0]);
  intptr_t nkeys = r->root_nkeys;
  for (;;) {
    if (unlikely(y->pg[r->level] != x->pg[r->level])) {
      ERROR("Mismatch cursors's pages at %zu level", r->level);
      return MDBX_PROBLEM;
    }
    r->diff = x->ki[r->level] - y->ki[r->level];
    if (r->diff)
      break;
    r->level += 1;
    if (r->level > depth) {
      r->diff = CMP2INT(x->flags & z_eof_hard, y->flags & z_eof_hard);
      return MDBX_SUCCESS;
    }
  }

  while (unlikely(r->diff == 1) && likely(r->level < depth)) {
    r->level += 1;
    /*   DB'PAGEs: 0------------------>MAX
     *
     *    CURSORs:       y < x
     *  STACK[i ]:         |
     *  STACK[+1]:  ...y++N|0++x...
     */
    nkeys = page_numkeys(y->pg[r->level]);
    r->diff = (nkeys - y->ki[r->level]) + x->ki[r->level];
    ASSERT(r->diff > 0);
  }

  while (unlikely(r->diff == -1) && likely(r->level < depth)) {
    r->level += 1;
    /*   DB'PAGEs: 0------------------>MAX
     *
     *    CURSORs:       x < y
     *  STACK[i ]:         |
     *  STACK[+1]:  ...x--N|0--y...
     */
    nkeys = page_numkeys(x->pg[r->level]);
    r->diff = -(nkeys - x->ki[r->level]) - y->ki[r->level];
    ASSERT(r->diff < 0);
  }

  return MDBX_SUCCESS;
}

__hot static ptrdiff_t estimate(const tree_t *tree, diff_t *const __restrict dr) {
  /*        root: branch-page    => scale = leaf-factor * branch-factor^(N-1)
   *     level-1: branch-page(s) => scale = leaf-factor * branch-factor^2
   *     level-2: branch-page(s) => scale = leaf-factor * branch-factor
   *     level-N: branch-page(s) => scale = leaf-factor
   *  leaf-level: leaf-page(s)   => scale = 1
   */
  ptrdiff_t btree_power = (ptrdiff_t)tree->height - 2 - (ptrdiff_t)dr->level;
  if (btree_power < 0)
    return dr->diff;

  ptrdiff_t estimated = (ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)tree->leaf_pages;
  if (btree_power == 0)
    return estimated;

  if (tree->height < 4) {
    ASSERT(dr->level == 0 && btree_power == 1);
    return (ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)dr->root_nkeys;
  }

  /* average_branchpage_fillfactor = total(branch_entries) / branch_pages
     total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */
  const size_t log2_fixedpoint = sizeof(size_t) - 1;
  const size_t half = UINT64_C(1) << (log2_fixedpoint - 1);
  const size_t factor = ((tree->leaf_pages + tree->branch_pages - 1) << log2_fixedpoint) / tree->branch_pages;
  while (1) {
    switch ((size_t)btree_power) {
    default: {
      const size_t square = (factor * factor + half) >> log2_fixedpoint;
      const size_t quad = (square * square + half) >> log2_fixedpoint;
      do {
        estimated = estimated * quad + half;
        estimated >>= log2_fixedpoint;
        btree_power -= 4;
      } while (btree_power >= 4);
      continue;
    }
    case 3:
      estimated = estimated * factor + half;
      estimated >>= log2_fixedpoint;
      __fallthrough /* fall through */;
    case 2:
      estimated = estimated * factor + half;
      estimated >>= log2_fixedpoint;
      __fallthrough /* fall through */;
    case 1:
      estimated = estimated * factor + half;
      estimated >>= log2_fixedpoint;
      __fallthrough /* fall through */;
    case 0:
      if (unlikely(estimated > (ptrdiff_t)tree->items))
        return (ptrdiff_t)tree->items;
      if (unlikely(estimated < -(ptrdiff_t)tree->items))
        return -(ptrdiff_t)tree->items;
      return estimated;
    }
  }
}

/*------------------------------------------------------------------------------
 * Range-Estimation API */

__hot int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, ptrdiff_t *distance_items) {
  if (unlikely(!distance_items))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = cursor_check_pure(first);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = cursor_check_pure(last);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  *distance_items = 0;
  diff_t dr;
  rc = cursor_diff(last, first, &dr);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cASSERT0(first, dr.diff || inner_pointed(first) == inner_pointed(last));
  if (unlikely(dr.diff == 0) && inner_pointed(first)) {
    first = &first->subcur->cursor;
    last = &last->subcur->cursor;
    rc = cursor_diff(first, last, &dr);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  if (likely(dr.diff != 0))
    *distance_items = estimate(first->tree, &dr);

  return MDBX_SUCCESS;
}

__hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op move_op,
                             ptrdiff_t *distance_items) {
  if (unlikely(!distance_items || move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = cursor_check_ro(cursor);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!is_pointed(cursor)))
    return LOG_IFERR(MDBX_ENODATA);

  cursor_couple_t next;
  rc = cursor_init(&next.outer, cursor->txn, cursor_dbi(cursor));
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_copy_position(cursor, &next.outer);

  MDBX_val stub_data;
  if (data == nullptr) {
    const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY;
    if (unlikely(mask & (1 << move_op)))
      return LOG_IFERR(MDBX_EINVAL);
    stub_data.iov_base = nullptr;
    stub_data.iov_len = 0;
    data = &stub_data;
  }

  MDBX_val stub_key;
  if (key == nullptr) {
    const unsigned mask =
        1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY | 1 << MDBX_SET | 1 << MDBX_SET_RANGE;
    if (unlikely(mask & (1 << move_op)))
      return LOG_IFERR(MDBX_EINVAL);
    stub_key.iov_base = nullptr;
    stub_key.iov_len = 0;
    key = &stub_key;
  }

  rc = cursor_ops(&next.outer, key, data, move_op);
  if (unlikely(rc != MDBX_SUCCESS && (rc != MDBX_NOTFOUND || !is_pointed(&next.outer))))
    return LOG_IFERR(rc);

  if (move_op == MDBX_LAST) {
    next.outer.flags |= z_eof_hard;
    /* coverity[UNINIT] */
    next.inner.cursor.flags |= z_eof_hard;
  }
  return mdbx_estimate_distance(cursor, &next.outer, distance_items);
}

__hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *begin_key, const MDBX_val *begin_data,
                              const MDBX_val *end_key, const MDBX_val *end_data, ptrdiff_t *size_items) {
  if (unlikely(!size_items))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(begin_data && (begin_key == nullptr || begin_key == MDBX_EPSILON)))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(end_data && (end_key == nullptr || end_key == MDBX_EPSILON)))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(begin_key == MDBX_EPSILON && end_key == MDBX_EPSILON))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t begin;
  /* LY: first, initialize cursor to refresh a DB in case it have DB_STALE */
  rc = cursor_init(&begin.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(begin.outer.tree->items == 0)) {
    *size_items = 0;
    return MDBX_SUCCESS;
  }

  if (!begin_key) {
    if (unlikely(!end_key)) {
      /* LY: FIRST..LAST case */
      *size_items = (ptrdiff_t)begin.outer.tree->items;
      return MDBX_SUCCESS;
    }
    rc = outer_first(&begin.outer, nullptr, nullptr);
    if (unlikely(end_key == MDBX_EPSILON)) {
      /* LY: FIRST..+epsilon case */
      return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc);
    }
  } else {
    if (unlikely(begin_key == MDBX_EPSILON)) {
      if (end_key == nullptr) {
        /* LY: -epsilon..LAST case */
        rc = outer_last(&begin.outer, nullptr, nullptr);
        return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc);
      }
      /* LY: -epsilon..value case */
      ASSERT(end_key != MDBX_EPSILON);
      begin_key = end_key;
    } else if (unlikely(end_key == MDBX_EPSILON)) {
      /* LY: value..+epsilon case */
      ASSERT(begin_key != MDBX_EPSILON);
      end_key = begin_key;
    }
    if (end_key && !begin_data && !end_data &&
        (begin_key == end_key || begin.outer.clc->k.cmp(begin_key, end_key) == 0)) {
      /* LY: single key case */
      rc = cursor_seek(&begin.outer, (MDBX_val *)begin_key, nullptr, MDBX_SET).err;
      if (unlikely(rc != MDBX_SUCCESS)) {
        *size_items = 0;
        return LOG_IFERR((rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc);
      }
      *size_items = 1;
      if (inner_pointed(&begin.outer))
        *size_items = (sizeof(*size_items) >= sizeof(begin.inner.nested_tree.items) ||
                       begin.inner.nested_tree.items <= PTRDIFF_MAX)
                          ? (size_t)begin.inner.nested_tree.items
                          : PTRDIFF_MAX;

      return MDBX_SUCCESS;
    } else {
      MDBX_val proxy_key = *begin_key;
      MDBX_val proxy_data = {nullptr, 0};
      if (begin_data)
        proxy_data = *begin_data;
      rc = LOG_IFERR(cursor_seek(&begin.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err);
    }
  }

  if (unlikely(rc != MDBX_SUCCESS)) {
    if (rc != MDBX_NOTFOUND || !is_pointed(&begin.outer))
      return LOG_IFERR(rc);
  }

  cursor_couple_t end;
  rc = cursor_init(&end.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  if (!end_key) {
    rc = outer_last(&end.outer, nullptr, nullptr);
    end.outer.flags |= z_eof_hard;
    end.inner.cursor.flags |= z_eof_hard;
  } else {
    MDBX_val proxy_key = *end_key;
    MDBX_val proxy_data = {nullptr, 0};
    if (end_data)
      proxy_data = *end_data;
    rc = cursor_seek(&end.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err;
  }
  if (unlikely(rc != MDBX_SUCCESS)) {
    if (rc != MDBX_NOTFOUND || !is_pointed(&end.outer))
      return LOG_IFERR(rc);
  }

  rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  ASSERT(*size_items >= -(ptrdiff_t)begin.outer.tree->items && *size_items <= (ptrdiff_t)begin.outer.tree->items);

#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation                                               \
       * results for an inverted ranges. */

  /* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63
     Change-Id: If59eccf7311123ab6384c4b93f9b1fed5a0a10d1 */

  if (*size_items < 0) {
    /* LY: inverted range case */
    *size_items += (ptrdiff_t)begin.outer.tree->items;
  } else if (*size_items == 0 && begin_key && end_key) {
    int cmp = begin.outer.kvx->cmp(&origin_begin_key, &origin_end_key);
    if (cmp == 0 && cursor_pointed(begin.inner.cursor.flags) &&
        begin_data && end_data)
      cmp = begin.outer.kvx->v.cmp(&origin_begin_data, &origin_end_data);
    if (cmp > 0) {
      /* LY: inverted range case with empty scope */
      *size_items = (ptrdiff_t)begin.outer.tree->items;
    }
  }
  ASSERT(*size_items >= 0 &&
         *size_items <= (ptrdiff_t)begin.outer.tree->items);
#endif

  return MDBX_SUCCESS;
}

__cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, uint32_t *mask) {
  if (unlikely(!mask))
    return LOG_IFERR(MDBX_EINVAL);
  *mask = 0;

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if ((cx.outer.tree->flags & MDBX_DUPSORT) == 0)
    return MDBX_RESULT_TRUE;

  MDBX_val key, data;
  rc = outer_first(&cx.outer, &key, &data);
  while (rc == MDBX_SUCCESS) {
    const node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
    const tree_t *db = node_data(node);
    const unsigned flags = node_flags(node);
    switch (flags) {
    case N_BIG:
    case 0:
      /* single-value entry, deep = 0 */
      *mask |= 1 << 0;
      break;
    case N_DUP:
      /* single sub-page, deep = 1 */
      *mask |= 1 << 1;
      break;
    case N_DUP | N_TREE:
      /* sub-tree */
      *mask |= 1 << UNALIGNED_PEEK_16(db, tree_t, height);
      break;
    default:
      ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node-size", flags);
      return LOG_IFERR(MDBX_CORRUPTED);
    }
    rc = outer_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP);
  }

  return LOG_IFERR((rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc);
}

int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) {
  if (unlikely(canary == nullptr))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
  if (unlikely(rc != MDBX_SUCCESS)) {
    memset(canary, 0, sizeof(*canary));
    return LOG_IFERR(rc);
  }

  *canary = txn->canary;
  return MDBX_SUCCESS;
}

int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) {
  DKBUF_DEBUG;
  DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));

  if (unlikely(!key || !data))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return LOG_IFERR(cursor_seek(&cx.outer, (MDBX_val *)key, data, MDBX_SET).err);
}

int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data) {
  if (unlikely(!key || !data))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  return LOG_IFERR(cursor_ops(&cx.outer, key, data, MDBX_SET_LOWERBOUND));
}

int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, size_t *values_count) {
  DKBUF_DEBUG;
  DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));

  if (unlikely(!key || !data))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = cursor_seek(&cx.outer, key, data, MDBX_SET_KEY).err;
  if (unlikely(rc != MDBX_SUCCESS)) {
    if (values_count)
      *values_count = 0;
    return LOG_IFERR(rc);
  }

  if (values_count) {
    *values_count = 1;
    if (inner_pointed(&cx.outer))
      *values_count =
          (sizeof(*values_count) >= sizeof(cx.inner.nested_tree.items) || cx.inner.nested_tree.items <= PTRDIFF_MAX)
              ? (size_t)cx.inner.nested_tree.items
              : PTRDIFF_MAX;
  }
  return MDBX_SUCCESS;
}

/*----------------------------------------------------------------------------*/

int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) {
  int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (likely(canary)) {
    if (txn->canary.x == canary->x && txn->canary.y == canary->y && txn->canary.z == canary->z)
      return MDBX_SUCCESS;
    txn->canary.x = canary->x;
    txn->canary.y = canary->y;
    txn->canary.z = canary->z;
  }
  txn->canary.v = txn->txnid;
  txn->flags |= MDBX_TXN_DIRTY;

  return MDBX_SUCCESS;
}

/* Функция сообщает находится ли указанный адрес в "грязной" странице у
 * заданной пишущей транзакции. В конечном счете это позволяет избавиться от
 * лишнего копирования данных из НЕ-грязных страниц.
 *
 * "Грязные" страницы - это те, которые уже были изменены в ходе пишущей
 * транзакции. Соответственно, какие-либо дальнейшие изменения могут привести
 * к перезаписи таких страниц. Поэтому все функции, выполняющие изменения, в
 * качестве аргументов НЕ должны получать указатели на данные в таких
 * страницах. В свою очередь "НЕ грязные" страницы перед модификацией будут
 * скопированы.
 *
 * Другими словами, данные из "грязных" страниц должны быть либо скопированы
 * перед передачей в качестве аргументов для дальнейших модификаций, либо
 * отвергнуты на стадии проверки корректности аргументов.
 *
 * Таким образом, функция позволяет как избавится от лишнего копирования,
 * так и выполнить более полную проверку аргументов.
 *
 * ВАЖНО: Передаваемый указатель должен указывать на начало данных. Только
 * так гарантируется что актуальный заголовок страницы будет физически
 * расположен в той-же странице памяти, в том числе для многостраничных
 * P_LARGE страниц с длинными данными. */
int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  const MDBX_env *env = txn->env;
  const ptrdiff_t offset = ptr_dist(ptr, env->dxb_mmap.base);
  if (offset >= 0) {
    const pgno_t pgno = bytes2pgno(env, offset);
    if (likely(pgno < txn->geo.first_unallocated)) {
      const page_t *page = pgno2page(env, pgno);
      if (unlikely(page->pgno != pgno || (page->flags & P_ILL_BITS) != 0)) {
        /* The ptr pointed into middle of a large page,
         * not to the beginning of a data. */
        return LOG_IFERR(MDBX_EINVAL);
      }
      return (page->txnid > txn_basis_snapshot(txn)) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
    }
    if ((size_t)offset < env->dxb_mmap.limit) {
      /* Указатель адресует что-то в пределах mmap, но за границей
       * распределенных страниц. Такое может случится если mdbx_is_dirty()
       * вызывается после операции, в ходе которой грязная страница была
       * возвращена в нераспределенное пространство. */
      return (txn->flags & txn_ro_flat) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE;
    }
  }

  /* Страница вне используемого mmap-диапазона, т.е. либо в функцию был
   * передан некорректный адрес, либо адрес в теневой странице, которая была
   * выделена посредством malloc().
   *
   * Для режима MDBX_WRITE_MAP режима страница однозначно "не грязная",
   * а для режимов без MDBX_WRITE_MAP однозначно "не чистая". */
  return (txn->flags & (MDBX_WRITEMAP | txn_ro_flat)) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE;
}

int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, const MDBX_val *data) {
  if (unlikely(!key))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(dbi <= FREE_DBI))
    return LOG_IFERR(MDBX_BAD_DBI);

  int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  MDBX_val proxy;
  MDBX_cursor_op op = MDBX_SET;
  unsigned flags = MDBX_ALLDUPS;
  if (data) {
    proxy = *data;
    data = &proxy;
    op = MDBX_GET_BOTH;
    flags = 0;
  }
  rc = cursor_seek(&cx.outer, (MDBX_val *)key, (MDBX_val *)data, op).err;
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cx.outer.next = txn->cursors[dbi];
  txn->cursors[dbi] = &cx.outer;
  rc = cursor_del(&cx.outer, flags);
  txn->cursors[dbi] = cx.outer.next;
  return LOG_IFERR(rc);
}

int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) {
  if (unlikely(!key || !data))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(dbi <= FREE_DBI))
    return LOG_IFERR(MDBX_BAD_DBI);

  if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND |
                         MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE)))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(flags & MDBX_MULTIPLE)) {
    rc = cursor_check_multiple(&cx.outer, key, data, flags);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  if (flags & MDBX_RESERVE) {
    if (unlikely(cx.outer.tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_INTEGERDUP | MDBX_DUPFIXED)))
      return LOG_IFERR(MDBX_INCOMPATIBLE);
    data->iov_base = nullptr;
  }

  cx.outer.next = txn->cursors[dbi];
  txn->cursors[dbi] = &cx.outer;

  /* LY: support for update (explicit overwrite) */
  if (flags & MDBX_CURRENT) {
    rc = cursor_seek(&cx.outer, (MDBX_val *)key, nullptr, MDBX_SET).err;
    if (likely(rc == MDBX_SUCCESS) && (txn->dbs[dbi].flags & MDBX_DUPSORT) && (flags & MDBX_ALLDUPS) == 0) {
      /* LY: allows update (explicit overwrite) only for unique keys */
      node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
      if (node_flags(node) & N_DUP) {
        cASSERT0(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1);
        rc = MDBX_EMULTIVAL;
        if ((flags & MDBX_NOOVERWRITE) == 0) {
          flags -= MDBX_CURRENT;
          rc = cursor_del(&cx.outer, MDBX_ALLDUPS);
        }
      }
    }
  }

  if (likely(rc == MDBX_SUCCESS))
    rc = cursor_put_checklen(&cx.outer, key, data, flags);
  txn->cursors[dbi] = cx.outer.next;

  return LOG_IFERR(rc);
}

//------------------------------------------------------------------------------

/* Позволяет обновить или удалить существующую запись с получением
 * в old_data предыдущего значения данных. При этом если new_data равен
 * нулю, то выполняется удаление, иначе обновление/вставка.
 *
 * Текущее значение может находиться в уже измененной (грязной) странице.
 * В этом случае страница будет перезаписана при обновлении, а само старое
 * значение утрачено. Поэтому исходно в old_data должен быть передан
 * дополнительный буфер для копирования старого значения.
 * Если переданный буфер слишком мал, то функция вернет -1, установив
 * old_data->iov_len в соответствующее значение.
 *
 * Для не-уникальных ключей также возможен второй сценарий использования,
 * когда посредством old_data из записей с одинаковым ключом для
 * удаления/обновления выбирается конкретная. Для выбора этого сценария
 * во flags следует одновременно указать MDBX_CURRENT и MDBX_NOOVERWRITE.
 * Именно эта комбинация выбрана, так как она лишена смысла, и этим позволяет
 * идентифицировать запрос такого сценария.
 *
 * Функция может быть замещена соответствующими операциями с курсорами
 * после двух доработок (TODO):
 *  - внешняя аллокация курсоров, в том числе на стеке (без malloc).
 *  - получения dirty-статуса страницы по адресу (знать о MUTABLE/WRITEABLE).
 */

int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data,
                    MDBX_put_flags_t flags, MDBX_preserve_func preserver, void *preserver_context) {
  if (unlikely(!key || !old_data || old_data == new_data))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(old_data->iov_base == nullptr && old_data->iov_len))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(new_data == nullptr && (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely(dbi <= FREE_DBI))
    return LOG_IFERR(MDBX_BAD_DBI);

  if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND |
                         MDBX_APPENDDUP | MDBX_CURRENT)))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  cursor_couple_t cx;
  rc = cursor_init(&cx.outer, txn, dbi);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  cx.outer.next = txn->cursors[dbi];
  txn->cursors[dbi] = &cx.outer;

  MDBX_val present_key = *key;
  if (F_ISSET(flags, MDBX_CURRENT | MDBX_NOOVERWRITE)) {
    /* в old_data значение для выбора конкретного дубликата */
    if (unlikely(!(txn->dbs[dbi].flags & MDBX_DUPSORT))) {
      rc = MDBX_EINVAL;
      goto bailout;
    }

    /* убираем лишний бит, он был признаком запрошенного режима */
    flags -= MDBX_NOOVERWRITE;

    rc = cursor_seek(&cx.outer, &present_key, old_data, MDBX_GET_BOTH).err;
    if (rc != MDBX_SUCCESS)
      goto bailout;
  } else {
    /* в old_data буфер для сохранения предыдущего значения */
    if (unlikely(new_data && old_data->iov_base == new_data->iov_base))
      return LOG_IFERR(MDBX_EINVAL);
    MDBX_val present_data;
    rc = cursor_seek(&cx.outer, &present_key, &present_data, MDBX_SET_KEY).err;
    if (unlikely(rc != MDBX_SUCCESS)) {
      old_data->iov_base = nullptr;
      old_data->iov_len = 0;
      if (rc != MDBX_NOTFOUND || (flags & MDBX_CURRENT))
        goto bailout;
    } else if (flags & MDBX_NOOVERWRITE) {
      rc = MDBX_KEYEXIST;
      *old_data = present_data;
      goto bailout;
    } else {
      page_t *page = cx.outer.pg[cx.outer.top];
      if (txn->dbs[dbi].flags & MDBX_DUPSORT) {
        if (flags & MDBX_CURRENT) {
          /* disallow update/delete for multi-values */
          node_t *node = page_node(page, cx.outer.ki[cx.outer.top]);
          if (node_flags(node) & N_DUP) {
            cASSERT0(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1);
            if (cx.outer.subcur->nested_tree.items > 1) {
              rc = MDBX_EMULTIVAL;
              goto bailout;
            }
          }
          /* В LMDB флажок MDBX_CURRENT здесь приведет
           * к замене данных без учета MDBX_DUPSORT сортировки,
           * но здесь это в любом случае допустимо, так как мы
           * проверили что для ключа есть только одно значение. */
        }
      }

      if (is_modifable(txn, page)) {
        if (new_data && eq_fast(&present_data, new_data)) {
          /* если данные совпадают, то ничего делать не надо */
          *old_data = *new_data;
          goto bailout;
        }
        rc = preserver ? preserver(preserver_context, old_data, present_data.iov_base, present_data.iov_len)
                       : MDBX_SUCCESS;
        if (unlikely(rc != MDBX_SUCCESS))
          goto bailout;
      } else {
        *old_data = present_data;
      }
      flags |= MDBX_CURRENT;
    }
  }

  if (likely(new_data))
    rc = cursor_put_checklen(&cx.outer, key, new_data, flags);
  else
    rc = cursor_del(&cx.outer, flags & MDBX_ALLDUPS);

bailout:
  txn->cursors[dbi] = cx.outer.next;
  return LOG_IFERR(rc);
}

static int default_value_preserver(void *context, MDBX_val *target, const void *src, size_t bytes) {
  (void)context;
  if (unlikely(target->iov_len < bytes)) {
    target->iov_base = nullptr;
    target->iov_len = bytes;
    return MDBX_RESULT_TRUE;
  }
  memcpy(target->iov_base, src, target->iov_len = bytes);
  return MDBX_SUCCESS;
}

int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data,
                 MDBX_put_flags_t flags) {
  return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags, default_value_preserver, nullptr);
}

static void txn_latency_init(MDBX_commit_latency *latency, struct commit_timestamp *ts) {
  uint64_t timestamp = 0;
  if (latency) {
    timestamp = osal_monotime();
    memset(latency, 0, sizeof(*latency));
  }
  ts->prep = ts->gc = ts->audit = ts->write = ts->sync = ts->start = timestamp;
  ts->gc_cpu = 0;
}

static void txn_latency_done(MDBX_commit_latency *latency, struct commit_timestamp *ts) {
  if (latency) {
    ASSERT(ts->start);
    latency->preparation = (ts->prep > ts->start) ? osal_monotime_to_16dot16(ts->prep - ts->start) : 0;
    latency->gc_wallclock = (ts->gc > ts->prep) ? osal_monotime_to_16dot16(ts->gc - ts->prep) : 0;
    latency->gc_cputime = ts->gc_cpu ? osal_monotime_to_16dot16(ts->gc_cpu) : 0;
    latency->audit = (ts->audit > ts->gc) ? osal_monotime_to_16dot16(ts->audit - ts->gc) : 0;
    latency->write = (ts->write > ts->audit) ? osal_monotime_to_16dot16(ts->write - ts->audit) : 0;
    latency->sync = (ts->sync > ts->write) ? osal_monotime_to_16dot16(ts->sync - ts->write) : 0;
    const uint64_t ts_end = osal_monotime();
    latency->ending = (ts_end > ts->sync) ? osal_monotime_to_16dot16(ts_end - ts->sync) : 0;
    latency->whole = osal_monotime_to_16dot16_noUnderflow(ts_end - ts->start);
  }
}

#ifdef __SANITIZE_THREAD__
/* LY: avoid tsan-trap by txn, mm_last_pg and geo.first_unallocated */
__attribute__((__no_sanitize_thread__, __noinline__))
#endif
int mdbx_txn_straggler(const MDBX_txn *txn, int *percent) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
  if (likely(rc == MDBX_SUCCESS))
    rc = check_env(txn->env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR((rc > 0) ? -rc : rc);

  if (unlikely((txn->flags & txn_ro_flat) == 0)) {
    if (percent)
      *percent = (int)((txn->geo.first_unallocated * UINT64_C(100) + txn->geo.end_pgno / 2) / txn->geo.end_pgno);
    return 0;
  }

  txnid_t lag;
  troika_t troika = meta_tap(txn->env);
  do {
    const meta_ptr_t head = meta_recent(txn->env, &troika);
    if (percent) {
      const pgno_t maxpg = head.ptr_v->geometry.now;
      *percent = (int)((head.ptr_v->geometry.first_unallocated * UINT64_C(100) + maxpg / 2) / maxpg);
    }
    lag = (head.txnid - txn->txnid) / xMDBX_TXNID_STEP;
  } while (unlikely(meta_should_retry(txn->env, &troika)));

  return (lag > INT_MAX) ? INT_MAX : (int)lag;
}

MDBX_env *mdbx_txn_env(const MDBX_txn *txn) {
  if (unlikely(!txn || txn->signature != txn_signature || txn->env->signature.weak != env_signature))
    return nullptr;
  return txn->env;
}

uint64_t mdbx_txn_id(const MDBX_txn *txn) {
  if (unlikely(!txn || txn->signature != txn_signature))
    return 0;
  return txn->txnid;
}

MDBX_txn_flags_t mdbx_txn_flags(const MDBX_txn *txn) {
  STATIC_ASSERT(
      (MDBX_TXN_INVALID & (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD |
                           txn_gc_drained | txn_shrink_allowed | txn_rw_begin_flags | txn_ro_begin_flags)) == 0);
  if (unlikely(!txn || txn->signature != txn_signature))
    return MDBX_TXN_INVALID;
  ASSERT(0 == (int)(txn->flags & MDBX_TXN_INVALID));

  MDBX_txn_flags_t flags = txn->flags;
  if (F_ISSET(flags, MDBX_TXN_PARKED | txn_ro_flat) && txn->ro.slot &&
      safe64_read(&txn->ro.slot->tid) == MDBX_TID_TXN_OUSTED)
    flags |= MDBX_TXN_OUSTED;
  if (flags & txn_ro_nested)
    flags += MDBX_TXN_RDONLY - txn_ro_nested;
  return flags;
}

int mdbx_txn_reset(MDBX_txn *txn) {
  int rc = check_txn(txn, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = check_env(txn->env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  /* This call is only valid for read-only txns */
  if (unlikely((txn->flags & txn_ro_flat) == 0))
    return LOG_IFERR(MDBX_EINVAL);

  return LOG_IFERR(txn_ro_reset(txn));
}

int mdbx_txn_break(MDBX_txn *txn) {
  do {
    int rc = check_txn(txn, 0);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
    txn->flags |= MDBX_TXN_ERROR;
    txn = txn->nested;
  } while (txn);
  return MDBX_SUCCESS;
}

int mdbx_txn_abort_ex(MDBX_txn *txn, MDBX_commit_latency *latency) {
  struct commit_timestamp ts;
  txn_latency_init(latency, &ts);

  int rc = check_txn(txn, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = check_env(txn->env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

#if MDBX_TXN_CHECKOWNER
  if ((txn->flags & (txn_ro_flat | MDBX_NOSTICKYTHREADS)) == MDBX_NOSTICKYTHREADS &&
      unlikely(txn->owner != osal_thread_self())) {
    mdbx_txn_break(txn);
    return LOG_IFERR(MDBX_THREAD_MISMATCH);
  }
#endif /* MDBX_TXN_CHECKOWNER */

  if (txn->nested) {
    /* more checks for middle-point abortion case */
    mdbx_txn_abort(txn->nested);
    cASSERT0(txn, !txn->nested);
  }

  rc = txn_abort(txn, latency);
  txn_latency_done(latency, &ts);
  return LOG_IFERR(rc);
}

int mdbx_txn_park(MDBX_txn *txn, bool autounpark) {
  STATIC_ASSERT(MDBX_TXN_BLOCKED > MDBX_TXN_ERROR);
  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = check_env(txn->env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely((txn->flags & txn_ro_flat) == 0))
    return LOG_IFERR(MDBX_TXN_INVALID);

  return LOG_IFERR(txn_ro_park(txn, autounpark));
}

int mdbx_txn_unpark(MDBX_txn *txn, bool restart_if_ousted) {
  STATIC_ASSERT(MDBX_TXN_BLOCKED > MDBX_TXN_PARKED + MDBX_TXN_ERROR);
  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED - MDBX_TXN_ERROR);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = check_env(txn->env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!F_ISSET(txn->flags, txn_ro_flat | MDBX_TXN_PARKED)))
    return MDBX_SUCCESS;

  rc = txn_ro_unpark(txn);
  if (likely(rc != MDBX_OUSTED) || !restart_if_ousted)
    return LOG_IFERR(rc);

  cASSERT0(txn, txn->flags & MDBX_TXN_FINISHED);
  rc = txn_ro_start(txn, false);
  return (rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : LOG_IFERR(rc);
}

int mdbx_txn_refresh(MDBX_txn *txn) {
  int rc = check_txn(txn, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = check_env(txn->env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely((txn->flags & txn_ro_flat) == 0))
    return LOG_IFERR(MDBX_EINVAL);

  if ((txn->flags & MDBX_TXN_FINISHED) == 0) {
    if (recent_committed_txnid(txn->env) == txn->txnid)
      return MDBX_RESULT_TRUE;
    rc = txn_ro_reset(txn);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }

  return LOG_IFERR(txn_ro_start(txn, false));
}

int mdbx_txn_renew(MDBX_txn *txn) {
  int rc = check_txn(txn, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  rc = check_env(txn->env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely((txn->flags & txn_ro_flat) == 0))
    return LOG_IFERR(MDBX_EINVAL);

  if (unlikely((txn->flags & MDBX_TXN_FINISHED) == 0)) {
    rc = txn_ro_reset(txn);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }

  rc = txn_ro_start(txn, false);
  if (rc == MDBX_SUCCESS) {
    tASSERT1(txn, txn->owner == (txn->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self());
    DEBUG("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid,
          (txn->flags & txn_ro_flat) ? 'r' : 'w', __Wpedantic_format_voidptr(txn), __Wpedantic_format_voidptr(txn->env),
          txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root);
  }
  return LOG_IFERR(rc);
}

int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) {
  int rc = check_txn(txn, 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  txn->userctx = ctx;
  return MDBX_SUCCESS;
}

void *mdbx_txn_get_userctx(const MDBX_txn *txn) { return check_txn(txn, MDBX_TXN_FINISHED) ? nullptr : txn->userctx; }

int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret, void *context) {
  if (unlikely(!ret))
    return LOG_IFERR(MDBX_EINVAL);
  *ret = nullptr;

  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(env->flags & MDBX_RDONLY & ~flags)) /* write txn in RDONLY env */
    return LOG_IFERR(MDBX_EACCESS);

  MDBX_txn *txn = nullptr;
  if (!parent) {
    if (flags & MDBX_TXN_RDONLY) {
      if (unlikely(flags & ~txn_ro_begin_flags))
        return LOG_IFERR(MDBX_EINVAL);
      txn = txn_alloc(txn_ro_flat, env);
      if (unlikely(!txn))
        return LOG_IFERR(MDBX_ENOMEM);
      rc = txn_ro_start(txn, F_ISSET(flags, MDBX_TXN_RDONLY_PREPARE));
      if (unlikely(rc != MDBX_SUCCESS)) {
        txn_ro_free(txn);
        return LOG_IFERR(rc);
      }
    } else {
      if (unlikely(flags & ~txn_rw_begin_flags))
        return LOG_IFERR(MDBX_EINVAL);
      rc = txn_basal_start(txn = env->basal_txn, flags);
      if (unlikely(rc != MDBX_SUCCESS))
        return LOG_IFERR(rc);
    }
  } else {
    if (unlikely(flags & ~(MDBX_TXN_READWRITE | MDBX_TXN_RDONLY)))
      return LOG_IFERR(MDBX_EINVAL);

    rc = check_txn(parent, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);

    if (unlikely(parent->flags & txn_ro_both)) {
      ERROR("%s", "Could not start a nested transaction from the flat read-only parent");
      return LOG_IFERR(MDBX_BAD_TXN);
    }

    if (unlikely(parent->env != env))
      return LOG_IFERR(MDBX_EINVAL);

    if (MDBX_ENABLE_FAKE_NESTED_READONLY_TRANSACTIONS && flags == MDBX_TXN_RDONLY) {
      txn = txn_nested_fakero_begin(parent);
      rc = txn ? MDBX_SUCCESS : MDBX_EINVAL;
    } else {
      rc = txn_nested_create(parent, (flags & MDBX_TXN_RDONLY) != 0);
      txn = parent->nested;
      if (unlikely(rc != MDBX_SUCCESS))
        return LOG_IFERR(rc);
    }
  }

  txn->signature = txn_signature;
  txn->userctx = context;

  if (F_ISSET(flags, MDBX_TXN_RDONLY_PREPARE))
    eASSERT0(env, txn->flags == (txn_ro_flat | MDBX_TXN_FINISHED));
  else if ((flags & txn_ro_flat) != 0 && !parent)
    eASSERT0(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | txn_ro_flat | MDBX_WRITEMAP |
                                  /* Win32: SRWL flag */ txn_shrink_allowed)) == 0);
  else {
    if (MDBX_ENABLE_FAKE_NESTED_READONLY_TRANSACTIONS)
      eASSERT0(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | txn_shrink_allowed | txn_may_have_cursors |
                                    MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC | MDBX_TXN_SPILLS | txn_ro_nested |
                                    MDBX_TXN_DIRTY | txn_gc_drained)) == 0);
    else {
      eASSERT0(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | txn_shrink_allowed | txn_may_have_cursors |
                                    MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC | MDBX_TXN_SPILLS | txn_ro_nested)) == 0);
      eASSERT0(env, !txn->wr.spilled.list && !txn->wr.spilled.least_removed);
    }
    if (CHECKS2_ENABLED())
      ENSURE_OBJ(txn, audit_ex(txn, 0, false) == 0);
  }

  *ret = txn;
  DEBUG("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid,
        (flags & txn_ro_flat) ? 'r' : 'w', __Wpedantic_format_voidptr(txn), __Wpedantic_format_voidptr(env),
        txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root);
  return MDBX_SUCCESS;
}

int mdbx_txn_commit_embark_read(MDBX_txn **ptxn, MDBX_commit_latency *latency) {
  struct commit_timestamp ts;
  txn_latency_init(latency, &ts);

  if (unlikely(!ptxn))
    return LOG_IFERR(MDBX_EINVAL);

  MDBX_txn *wtxn = *ptxn;
  int rc = check_txn(wtxn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  MDBX_env *const env = wtxn->env;
  rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

#if MDBX_TXN_CHECKOWNER
  if ((wtxn->flags & MDBX_NOSTICKYTHREADS) && wtxn == env->basal_txn && unlikely(wtxn->owner != osal_thread_self())) {
    mdbx_txn_break(wtxn);
    return LOG_IFERR(MDBX_THREAD_MISMATCH);
  }
#endif /* MDBX_TXN_CHECKOWNER */

  if (unlikely(wtxn->flags & txn_ro_both))
    goto done;

  if (unlikely(wtxn->nested)) {
    /* more checks for middle-point committing case */
    rc = mdbx_txn_commit_ex(wtxn->nested, nullptr);
    cASSERT0(wtxn, !wtxn->nested);
    if (unlikely(rc != MDBX_SUCCESS)) {
      *ptxn = nullptr;
      mdbx_txn_abort(wtxn);
      goto done;
    }
  }

  MDBX_txn *rtxn = nullptr;
  if (wtxn == env->basal_txn) {
    void *const preserved_context = wtxn->userctx;
    rc = txn_basal_commit(wtxn, latency ? &ts : nullptr);
    if (likely(rc == MDBX_SUCCESS)) {
      rc = txn_basal_end(wtxn, /* завершение транзакции без освобождения блокировки */ false);
      if (likely(rc == MDBX_SUCCESS)) {
        rtxn = txn_alloc(txn_ro_flat, env);
        rc = likely(rtxn) ? txn_ro_start(rtxn, false) : MDBX_ENOMEM;
      }
    }

    const int err = txn_basal_end(wtxn, /* освобождение блокировки */ true);
    if (unlikely(err != MDBX_SUCCESS) && rc == MDBX_SUCCESS)
      rc = err;
    if (likely(rc == MDBX_SUCCESS)) {
      rtxn->userctx = preserved_context;
      rtxn->signature = txn_signature;
    } else if (rtxn) {
      txn_ro_free(rtxn);
      rtxn = nullptr;
    }
  } else {
    rc = txn_nested_checkpoint(wtxn, latency ? &ts : nullptr);
    if (likely(rc == MDBX_SUCCESS)) {
      rtxn = wtxn;
      rtxn->flags |= txn_ro_nested;
      cASSERT0(rtxn, rtxn->env->txn == rtxn);
    }
  }
  *ptxn = rtxn;

done:
  txn_latency_done(latency, &ts);
  return LOG_IFERR(rc);
}

int mdbx_txn_checkpoint(MDBX_txn *txn, MDBX_txn_flags_t weakening_durability, MDBX_commit_latency *latency) {
  struct commit_timestamp ts;
  txn_latency_init(latency, &ts);

  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(weakening_durability & ~(MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_NOWEAKING)))
    return LOG_IFERR(MDBX_EINVAL);

  MDBX_env *const env = txn->env;
  rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

#if MDBX_TXN_CHECKOWNER
  if ((txn->flags & MDBX_NOSTICKYTHREADS) && txn == env->basal_txn && unlikely(txn->owner != osal_thread_self())) {
    mdbx_txn_break(txn);
    return LOG_IFERR(MDBX_THREAD_MISMATCH);
  }
#endif /* MDBX_TXN_CHECKOWNER */

  if (unlikely(txn->nested)) {
    /* more checks for middle-point committing case */
    rc = mdbx_txn_commit_ex(txn->nested, nullptr);
    cASSERT0(txn, !txn->nested);
    if (unlikely(rc != MDBX_SUCCESS)) {
      mdbx_txn_abort(txn);
      goto done;
    }
  }

  if ((txn->flags & MDBX_TXN_DIRTY) == 0)
    return MDBX_RESULT_TRUE;

  if (txn == env->basal_txn) {
    rc = txn_basal_checkpoint(txn, weakening_durability, latency ? &ts : nullptr);
  } else {
    if (unlikely(weakening_durability != MDBX_TXN_NOWEAKING))
      return LOG_IFERR(MDBX_EINVAL);
    rc = txn_nested_checkpoint(txn, latency ? &ts : nullptr);
  }

done:
  txn_latency_done(latency, &ts);
  return LOG_IFERR(rc);
}

int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) {
  STATIC_ASSERT(MDBX_TXN_FINISHED == MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD - MDBX_TXN_ERROR - MDBX_TXN_PARKED);

  struct commit_timestamp ts;
  txn_latency_init(latency, &ts);

  int rc = check_txn(txn, MDBX_TXN_FINISHED);
  if (unlikely(rc != MDBX_SUCCESS)) {
    if (rc == MDBX_BAD_TXN && F_ISSET(txn->flags, MDBX_TXN_FINISHED | txn_ro_flat)) {
      rc = MDBX_RESULT_TRUE;
      mdbx_txn_abort(txn);
      goto done;
    }
    return LOG_IFERR(rc);
  }

  MDBX_env *const env = txn->env;
  rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

#if MDBX_TXN_CHECKOWNER
  if ((txn->flags & MDBX_NOSTICKYTHREADS) && txn == env->basal_txn && unlikely(txn->owner != osal_thread_self())) {
    mdbx_txn_break(txn);
    return LOG_IFERR(MDBX_THREAD_MISMATCH);
  }
#endif /* MDBX_TXN_CHECKOWNER */

  if (unlikely(txn->nested)) {
    /* more checks for middle-point committing case */
    rc = mdbx_txn_commit_ex(txn->nested, nullptr);
    cASSERT0(txn, !txn->nested);
    if (unlikely(rc != MDBX_SUCCESS)) {
      mdbx_txn_abort(txn);
      goto done;
    }
  }

  rc = txn_commit(txn, latency, latency ? &ts : nullptr);
done:
  txn_latency_done(latency, &ts);
  return LOG_IFERR(rc);
}

int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) {
  int rc = check_txn(txn, MDBX_TXN_FINISHED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely(!info))
    return LOG_IFERR(MDBX_EINVAL);

  MDBX_env *const env = txn->env;
#if MDBX_ENV_CHECKPID
  if (unlikely(env->pid != osal_getpid())) {
    env->flags |= ENV_FATAL_ERROR;
    return LOG_IFERR(MDBX_PANIC);
  }
#endif /* MDBX_ENV_CHECKPID */

  info->txn_id = txn->txnid;
  info->txn_space_used = pgno2bytes(env, txn->geo.first_unallocated);
  info->txn_pget =
#if MDBX_ENABLE_PGET_STAT
      txn->ops_pget
#else
      0
#endif /* MDBX_ENABLE_PGET_STAT */
      ;

  if (txn->flags & txn_ro_flat) {
    meta_ptr_t head;
    uint64_t head_retired;
    troika_t troika = meta_tap(env);
    do {
      /* fetch info from volatile head */
      head = meta_recent(env, &troika);
      head_retired = unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired);
      info->txn_space_limit_soft = pgno2bytes(env, head.ptr_v->geometry.now);
      info->txn_space_limit_hard = pgno2bytes(env, head.ptr_v->geometry.upper);
      info->txn_space_leftover = pgno2bytes(env, head.ptr_v->geometry.now - head.ptr_v->geometry.first_unallocated);
    } while (unlikely(meta_should_retry(env, &troika)));

    info->txn_reader_lag = head.txnid - info->txn_id;
    info->txn_space_dirty = info->txn_space_retired = 0;
    uint64_t reader_snapshot_pages_retired = 0;
    if (txn->ro.slot &&
        ((txn->flags & MDBX_TXN_PARKED) == 0 || safe64_read(&txn->ro.slot->tid) != MDBX_TID_TXN_OUSTED) &&
        head_retired >
            (reader_snapshot_pages_retired = atomic_load64(&txn->ro.slot->snapshot_pages_retired, mo_Relaxed))) {
      info->txn_space_dirty = info->txn_space_retired =
          pgno2bytes(env, (pgno_t)(head_retired - reader_snapshot_pages_retired));

      size_t retired_next_reader = 0;
      lck_t *const lck = env->lck_mmap.lck;
      if (scan_rlt && info->txn_reader_lag > 1 && lck) {
        /* find next more recent reader */
        txnid_t next_reader = head.txnid;
        const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
        for (size_t i = 0; i < snap_nreaders; ++i) {
        retry:
          if (atomic_load_pid(&lck->rdt[i].pid, mo_AcquireRelease)) {
            jitter4testing(true);
            const uint64_t snap_tid = safe64_read(&lck->rdt[i].tid);
            const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid);
            const uint64_t snap_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_AcquireRelease);
            if (unlikely(snap_retired != atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed)) ||
                snap_txnid != safe64_read(&lck->rdt[i].txnid) || snap_tid != safe64_read(&lck->rdt[i].tid))
              goto retry;
            if (snap_txnid <= txn->txnid) {
              retired_next_reader = 0;
              break;
            }
            if (snap_txnid < next_reader && snap_tid >= MDBX_TID_TXN_OUSTED) {
              next_reader = snap_txnid;
              retired_next_reader = pgno2bytes(
                  env, (pgno_t)(snap_retired - atomic_load64(&txn->ro.slot->snapshot_pages_retired, mo_Relaxed)));
            }
          }
        }
      }
      info->txn_space_dirty = retired_next_reader;
    }
  } else {
    info->txn_space_limit_soft = pgno2bytes(env, txn->geo.now);
    info->txn_space_limit_hard = pgno2bytes(env, txn->geo.upper);
    info->txn_space_retired =
        pgno2bytes(env, txn->nested ? (size_t)txn->wr.retired_pages : pnl_size(txn->wr.retired_pages));
    info->txn_space_leftover = pgno2bytes(env, txn->wr.dirtyroom);
    info->txn_space_dirty =
        pgno2bytes(env, txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose
                                          : (txn->wr.writemap_dirty_npages + txn->wr.writemap_spilled_npages));
    info->txn_reader_lag = INT64_MAX;
    lck_t *const lck = env->lck_mmap.lck;
    if (scan_rlt && lck) {
      txnid_t oldest_reading = txn->txnid;
      const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
      if (snap_nreaders) {
        txn_gc_detent(txn);
        oldest_reading = txn->env->gc.detent;
        if (oldest_reading == txn->wr.troika.txnid[txn->wr.troika.recent]) {
          /* Если самый старый используемый снимок является предыдущим, т.е. непосредственно предшествующим текущей
           * транзакции, то просматриваем таблицу читателей чтобы выяснить действительно ли снимок используется
           * читателями. */
          oldest_reading = txn->txnid;
          for (size_t i = 0; i < snap_nreaders; ++i) {
            if (atomic_load_pid(&lck->rdt[i].pid, mo_Relaxed) &&
                txn->env->gc.detent == safe64_read(&lck->rdt[i].txnid)) {
              oldest_reading = txn->env->gc.detent;
              break;
            }
          }
        }
      }
      info->txn_reader_lag = txn->txnid - oldest_reading;
    }
  }

  return MDBX_SUCCESS;
}

int mdbx_txn_clone(const MDBX_txn *source, MDBX_txn **in_out_clone, void *clone_context) {
  if (unlikely(!in_out_clone))
    return LOG_IFERR(MDBX_EINVAL);

  MDBX_txn *clone = *in_out_clone;
retry:
  osal_memory_fence(mo_AcquireRelease, true);
  int rc = check_txn_anythread(
      source,
      MDBX_TXN_FINISHED |
          /* there is no difficulty, but we do not allow dirty transactions to avoid confusion */ MDBX_TXN_DIRTY);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

  if (unlikely(source->parent)) {
    /* do not immediately jump to env->basal_txn,
     * but check for the absence of MDBX_TXN_DIRTY in all parent transactions. */
    source = source->parent;
    goto retry;
  }

  MDBX_env *const env = source->env;
  rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

  const txnid_t txnid = source->txnid;
  if (clone) {
    rc = check_txn(clone, 0);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;
    if (unlikely((clone->flags & txn_ro_flat) == 0)) {
      rc = MDBX_EINVAL;
      goto bailout;
    }

    if (clone_context == clone)
      clone_context = clone->userctx;

    if (unlikely((clone->flags & MDBX_TXN_FINISHED) == 0)) {
      rc = txn_ro_reset(clone);
      if (unlikely(rc != MDBX_SUCCESS))
        goto bailout;
      goto retry;
    }
  } else {
    clone = txn_alloc(txn_ro_flat | MDBX_TXN_FINISHED, env);
    if (unlikely(!clone))
      return LOG_IFERR(MDBX_ENOMEM);
    clone->signature = txn_signature;
    goto retry;
  }

  rc = txn_ro_clone(source, clone);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

  osal_memory_fence(mo_AcquireRelease, true);
  rc = check_txn_anythread(source, MDBX_TXN_FINISHED | MDBX_TXN_DIRTY);
  if (unlikely(rc != MDBX_SUCCESS))
    goto bailout;

  clone->flags &= ~MDBX_TXN_FINISHED;
  clone->userctx = clone_context;
  if (unlikely(txn_basis_snapshot(source) != clone->txnid || txnid != source->txnid)) {
    rc = MDBX_PROBLEM;
    ERROR("unexpected mvcc-tnxid (%" PRIu64 " != %" PRIu64 ") mismatch during cloning of a transaction,"
          " seems the original transaction competitively restarted in another thread",
          txnid, (txnid != source->txnid) ? source->txnid : clone->txnid);
    goto bailout;
  }

  *in_out_clone = clone;
  return MDBX_SUCCESS;

bailout:
  if (clone) {
    if (clone != *in_out_clone)
      txn_ro_free(clone);
    else
      txn_ro_reset(clone);
  }
  return LOG_IFERR(rc);
}

int mdbx_txn_rollback(MDBX_txn *txn) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR - MDBX_TXN_PARKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  txn->flags |= /* avoid merge cursors' state */ MDBX_TXN_ERROR;
  if (txn->flags & txn_may_have_cursors)
    txn_done_cursors(txn);

  if (likely(txn->flags & txn_ro_flat) == 0) {
    if (!txn->parent)
      return LOG_IFERR(txn_basal_rollback(txn));
    else
      return LOG_IFERR(txn_nested_rollback(txn));
  }

  rc = txn_ro_reset(txn);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  return LOG_IFERR(txn_ro_start(txn, false));
}

int mdbx_txn_amend(MDBX_txn *rtxn, MDBX_txn **ptxn, MDBX_txn_flags_t flags, void *context) {
  if (unlikely(!ptxn))
    return LOG_IFERR(MDBX_EINVAL);

  if (*ptxn != rtxn)
    *ptxn = nullptr;

  if (unlikely(flags & ~(txn_rw_begin_flags | MDBX_TXN_RDONLY_PREPARE)))
    return LOG_IFERR(MDBX_EINVAL);

  int rc = check_txn(rtxn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  MDBX_env *const env = rtxn->env;
  rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  if (unlikely((rtxn->flags & txn_ro_flat) == 0))
    return LOG_IFERR(MDBX_BAD_TXN);

  /* avoid recursive deadlock */
  if (env->txn && env->basal_txn->owner == osal_thread_self())
    return LOG_IFERR(MDBX_TXN_OVERLAPPING);

  /* first check that there were no new commits */
  if (recent_committed_txnid(env) != rtxn->txnid)
    return LOG_IFERR(MDBX_RESULT_TRUE);

  STATIC_ASSERT(MDBX_TXN_TRY != MDBX_TXN_PARKED);
  if ((uint32_t)(flags & MDBX_TXN_TRY) == (uint32_t)(rtxn->flags & MDBX_TXN_PARKED)) {
    rc = txn_ro_park(rtxn, true);
    if (unlikely(rc != MDBX_SUCCESS))
      return LOG_IFERR(rc);
  }
  rc = lck_txn_lock(env, (flags & MDBX_TXN_TRY) != 0);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  /* after the lock was acuired we re-check that there were no new commits */
  if (recent_committed_txnid(env) != rtxn->txnid) {
    lck_txn_unlock(env);
    return LOG_IFERR(MDBX_RESULT_TRUE);
  }

  rc = txn_basal_start(env->basal_txn, (flags & ~MDBX_TXN_RDONLY_PREPARE) | txn_rw_already_locked);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);

  /* A paranoid check that changes will be made on top of the desired data snapshot */
  rc = MDBX_RESULT_TRUE;
  if (txn_basis_snapshot(env->basal_txn) == rtxn->txnid)
    rc = txn_ro_reset(rtxn);
  if (unlikely(rc != MDBX_SUCCESS)) {
    env->basal_txn->flags |= MDBX_TXN_ERROR;
    txn_basal_end(env->basal_txn, true);
    return LOG_IFERR(rc);
  }

  if (!F_ISSET(flags, MDBX_TXN_RDONLY_PREPARE))
    txn_ro_free(rtxn);
  env->basal_txn->userctx = context;
  *ptxn = env->basal_txn;
  return MDBX_SUCCESS;
}

struct audit_ctx {
  size_t used;
  uint8_t *const done_bitmap;
};

static int audit_dbi(void *ctx, const MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags,
                     const struct MDBX_stat *stat, MDBX_dbi dbi) {
  struct audit_ctx *audit_ctx = ctx;
  (void)name;
  (void)txn;
  (void)flags;
  audit_ctx->used += (size_t)stat->ms_branch_pages + (size_t)stat->ms_leaf_pages + (size_t)stat->ms_overflow_pages;
  if (dbi)
    audit_ctx->done_bitmap[dbi / CHAR_BIT] |= 1 << dbi % CHAR_BIT;
  return MDBX_SUCCESS;
}

static size_t audit_db_used(const tree_t *db) {
  return db ? (size_t)db->branch_pages + (size_t)db->leaf_pages + (size_t)db->large_pages : 0;
}

__cold static int audit_ex_locked(MDBX_txn *txn, const size_t retired_stored, const bool dont_filter_gc) {
  const MDBX_env *const env = txn->env;
  cASSERT0(txn, (txn->flags & txn_ro_flat) == 0);
  const size_t pending =
      txn->wr.loose_count + pnl_size(txn->wr.repnl) + (pnl_size(txn->wr.retired_pages) - retired_stored);

  cursor_couple_t cx;
  int rc = cursor_init(&cx.outer, txn, FREE_DBI);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  size_t gc = 0;
  MDBX_val key, data;
  rc = outer_first(&cx.outer, &key, &data);
  while (rc == MDBX_SUCCESS) {
    if (unlikely(key.iov_len != sizeof(txnid_t))) {
      ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len);
      return MDBX_CORRUPTED;
    }
    const txnid_t id = unaligned_peek_u64(4, key.iov_base);
    const size_t len = *(pgno_t *)data.iov_base;
    const bool acc = dont_filter_gc || !gc_is_reclaimed(txn, id);
    TRACE("%s id %" PRIaTXN " len %zu", acc ? "acc" : "skip", id, len);
    if (acc)
      gc += len;
    rc = outer_next(&cx.outer, &key, &data, MDBX_NEXT);
  }
  if (rc != MDBX_NOTFOUND)
    ERROR("unexpected gc-cursor rc %d, gc %zu", rc, gc);
  cASSERT0(txn, rc == MDBX_NOTFOUND);

  const size_t done_bitmap_size = (txn->n_dbi + CHAR_BIT - 1) / CHAR_BIT;
  if (txn->parent) {
    cASSERT0(txn, txn->n_dbi == txn->parent->n_dbi && txn->n_dbi == txn->env->txn->n_dbi);
#if MDBX_ENABLE_DBI_SPARSE
    cASSERT0(txn, txn->dbi_sparse == txn->parent->dbi_sparse && txn->dbi_sparse == txn->env->txn->dbi_sparse);
#endif /* MDBX_ENABLE_DBI_SPARSE */
  }

  struct audit_ctx ctx = {0, alloca(done_bitmap_size)};
  memset(ctx.done_bitmap, 0, done_bitmap_size);
  ctx.used =
      NUM_METAS + audit_db_used(dbi_dig(txn, FREE_DBI, nullptr)) + audit_db_used(dbi_dig(txn, MAIN_DBI, nullptr));

  rc = mdbx_enumerate_tables(txn, audit_dbi, &ctx);
  cASSERT0(txn, rc == MDBX_SUCCESS);

  for (size_t dbi = CORE_DBS; dbi < txn->n_dbi; ++dbi) {
    if (ctx.done_bitmap[dbi / CHAR_BIT] & (1 << dbi % CHAR_BIT))
      continue;
    const tree_t *db = dbi_dig(txn, dbi, nullptr);
    if (db)
      ctx.used += audit_db_used(db);
    else if (dbi_state(txn, dbi))
      WARNING("audit %s@%" PRIaTXN ": unable account dbi %zd / \"%.*s\", state 0x%02x", txn->parent ? "nested-" : "",
              txn->txnid, dbi, (int)env->kvs[dbi].name.iov_len, (const char *)env->kvs[dbi].name.iov_base,
              dbi_state(txn, dbi));
  }

  if (pending + gc + ctx.used == txn->geo.first_unallocated)
    return MDBX_SUCCESS;

  if ((txn->flags & txn_ro_flat) == 0)
    ERROR("audit @%" PRIaTXN ": %zu(pending) = %zu(loose) + "
          "%zu(reclaimed) + %zu(retired-pending) - %zu(retired-stored)",
          txn->txnid, pending, txn->wr.loose_count, pnl_size(txn->wr.repnl),
          txn->wr.retired_pages ? pnl_size(txn->wr.retired_pages) : 0, retired_stored);
  ERROR("audit @%" PRIaTXN ": %zu(pending) + %zu"
        "(gc) + %zu(count) = %zu(total) <> %zu"
        "(allocated)",
        txn->txnid, pending, gc, ctx.used, pending + gc + ctx.used, (size_t)txn->geo.first_unallocated);
  return MDBX_PROBLEM;
}

__cold int audit_ex(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc) {
  MDBX_env *const env = txn->env;
  int rc = osal_fastmutex_acquire(&env->dbi_lock);
  if (likely(rc == MDBX_SUCCESS)) {
    const unsigned preserve_txn_flags = txn->flags;
    txn->flags &= ~MDBX_TXN_BLOCKED;
    rc = audit_ex_locked(txn, retired_stored, dont_filter_gc);
    txn->flags = preserve_txn_flags;
    ENSURE_OBJ(env, osal_fastmutex_release(&env->dbi_lock) == MDBX_SUCCESS);
  }
  return rc;
}

typedef struct MDBX_chk_internal {
  MDBX_chk_context_t *usr;
  const struct MDBX_chk_callbacks *cb;
  uint64_t monotime_timeout;

  size_t *problem_counter;
  uint8_t flags;
  bool got_break;
  bool write_locked;
  uint8_t scope_depth;
  pgno_t last_nested_root;

  MDBX_chk_table_t table_gc, table_main;
  int16_t *pagemap;
  MDBX_chk_table_t *last_lookup;
  MDBX_chk_scope_t scope_stack[12];
  MDBX_chk_table_t *table[MDBX_MAX_DBI + CORE_DBS];

  MDBX_envinfo envinfo;
  troika_t troika;
  MDBX_val v2a_buf;
} MDBX_chk_internal_t;

__cold int chk_check_break(MDBX_chk_scope_t *const scope) {
  MDBX_chk_internal_t *const chk = scope->internal;
  return (chk->got_break || (chk->cb->check_break && (chk->got_break = chk->cb->check_break(chk->usr))))
             ? MDBX_RESULT_TRUE
             : MDBX_RESULT_FALSE;
}

__cold void chk_line_end(MDBX_chk_line_t *line) {
  if (likely(line)) {
    MDBX_chk_internal_t *chk = line->ctx->internal;
    ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
    if (likely(chk->cb->print_done))
      chk->cb->print_done(line);
  }
}

__cold MDBX_chk_line_t *chk_line_begin(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity) {
  MDBX_chk_internal_t *const chk = scope->internal;
  if (severity < MDBX_chk_warning)
    mdbx_env_chk_encount_problem(chk->usr);
  MDBX_chk_line_t *line = nullptr;
  if (likely(chk->cb->print_begin)) {
    line = chk->cb->print_begin(chk->usr, severity);
    if (likely(line)) {
      ASSERT(line->ctx == nullptr || (line->ctx == chk->usr && line->empty));
      ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
      line->ctx = chk->usr;
    }
  }
  return line;
}

__cold MDBX_chk_line_t *chk_line_feed(MDBX_chk_line_t *line) {
  if (likely(line)) {
    MDBX_chk_internal_t *chk = line->ctx->internal;
    enum MDBX_chk_severity severity = line->severity;
    chk_line_end(line);
    line = chk_line_begin(chk->usr->scope, severity);
  }
  return line;
}

__cold MDBX_chk_line_t *chk_flush(MDBX_chk_line_t *line) {
  if (likely(line)) {
    MDBX_chk_internal_t *chk = line->ctx->internal;
    ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
    if (likely(chk->cb->print_flush)) {
      chk->cb->print_flush(line);
      ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
      line->out = line->begin;
    }
  }
  return line;
}

__cold size_t chk_print_wanna(MDBX_chk_line_t *line, size_t need) {
  if (likely(line && need)) {
    size_t have = line->end - line->out;
    ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
    if (need > have) {
      line = chk_flush(line);
      have = line->end - line->out;
    }
    return (need < have) ? need : have;
  }
  return 0;
}

__cold MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, const char *str) {
  if (likely(line && str && *str)) {
    MDBX_chk_internal_t *chk = line->ctx->internal;
    size_t left = strlen(str);
    ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
    if (chk->cb->print_chars) {
      chk->cb->print_chars(line, str, left);
      ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
    } else
      do {
        size_t chunk = chk_print_wanna(line, left);
        ASSERT(chunk <= left);
        if (unlikely(!chunk))
          break;
        memcpy(line->out, str, chunk);
        line->out += chunk;
        ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
        str += chunk;
        left -= chunk;
      } while (left);
    line->empty = false;
  }
  return line;
}

__cold MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, const char *fmt, va_list args) {
  if (likely(line)) {
    MDBX_chk_internal_t *chk = line->ctx->internal;
    ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
    if (chk->cb->print_format) {
      chk->cb->print_format(line, fmt, args);
      ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
    } else {
      va_list ones;
      va_copy(ones, args);
      const int needed = vsnprintf(nullptr, 0, fmt, ones);
      va_end(ones);
      if (likely(needed > 0)) {
        const size_t have = chk_print_wanna(line, needed);
        if (likely(have > 0)) {
          int written = vsnprintf(line->out, have, fmt, args);
          if (likely(written > 0))
            line->out += written;
          ASSERT(line->begin <= line->end && line->begin <= line->out && line->out <= line->end);
        }
      }
    }
    line->empty = false;
  }
  return line;
}

__cold MDBX_chk_line_t *MDBX_PRINTF_ARGS(2, 3) chk_print(MDBX_chk_line_t *line, const char *fmt, ...) {
  if (likely(line)) {
    // MDBX_chk_internal_t *chk = line->ctx->internal;
    va_list args;
    va_start(args, fmt);
    line = chk_print_va(line, fmt, args);
    va_end(args);
    line->empty = false;
  }
  return line;
}

MDBX_MAYBE_UNUSED __cold void chk_println_va(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity,
                                             const char *fmt, va_list args) {
  chk_line_end(chk_print_va(chk_line_begin(scope, severity), fmt, args));
}

MDBX_MAYBE_UNUSED __cold void chk_println(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity,
                                          const char *fmt, ...) {
  va_list args;
  va_start(args, fmt);
  chk_println_va(scope, severity, fmt, args);
  va_end(args);
}

__cold static MDBX_chk_line_t *chk_print_size(MDBX_chk_line_t *line, const char *prefix, const uint64_t value,
                                              const char *suffix) {
  static const char sf[] = "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */
  if (likely(line)) {
    MDBX_chk_internal_t *chk = line->ctx->internal;
    prefix = prefix ? prefix : "";
    suffix = suffix ? suffix : "";
    if (chk->cb->print_size)
      chk->cb->print_size(line, prefix, value, suffix);
    else
      for (unsigned i = 0;; ++i) {
        const unsigned scale = 10 + i * 10;
        const uint64_t rounded = value + (UINT64_C(5) << (scale - 10));
        const uint64_t integer = rounded >> scale;
        const uint64_t fractional = (rounded - (integer << scale)) * 100u >> scale;
        if ((rounded >> scale) <= 1000)
          return chk_print(line, "%s%" PRIu64 " (%u.%02u %ciB)%s", prefix, value, (unsigned)integer,
                           (unsigned)fractional, sf[i], suffix);
      }
    line->empty = false;
  }
  return line;
}

__cold static MDBX_chk_line_t *chk_print_ratio(MDBX_chk_line_t *line, size_t numerator, size_t denominator,
                                               unsigned precision) {
  if (line) {
    ratio2digits_buffer_t buffer;
    line = chk_puts(line, ratio2digits(numerator, denominator, &buffer, precision));
  }
  return line;
}

__cold static MDBX_chk_line_t *chk_print_percent(MDBX_chk_line_t *line, const char *triplet, size_t value, size_t whole,
                                                 const char *unit) {
  if (line) {
    const char *s1 = triplet;
    const char *s2 = s1 + strlen(s1) + 1;
    const char *s3 = s2 + strlen(s2) + 1;
    ratio2digits_buffer_t buffer;
    line = chk_print(line, "%s %" PRIuSIZE "%s%s (%s%%%s)%s", s1, value, unit, &"s"[*unit == 0 || value == 1],
                     ratio2percent(value, whole, &buffer), s2, s3);
  }
  return line;
}

__cold static MDBX_chk_line_t *chk_print_pages_percent(MDBX_chk_line_t *line, const char *triplet, size_t pages,
                                                       size_t whole) {
  return chk_print_percent(line, triplet, pages, whole, " page");
}

__cold static MDBX_chk_line_t *chk_print_bytes_percent(MDBX_chk_line_t *line, const char *triplet, size_t pages,
                                                       size_t whole) {
  return chk_print_percent(line, triplet, pages, whole, " byte");
}

__cold static MDBX_chk_line_t *chk_print_pages_percent_bb(MDBX_chk_line_t *line, const char *prefix, size_t pages,
                                                          size_t backed, size_t boundary) {
  if (line) {
    ratio2digits_buffer_t buffer_backed, buffer_boundary;
    line =
        chk_print(line, "%s %" PRIuSIZE " page%s (%s%% of backed, %s%% of boundary)", prefix, pages, &"s"[pages == 1],
                  ratio2percent(pages, backed, &buffer_backed), ratio2percent(pages, boundary, &buffer_boundary));
  }
  return line;
}

__cold static int chk_error_rc(MDBX_chk_scope_t *const scope, int err, const char *subj) {
  MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_error);
  MDBX_chk_internal_t *const chk = scope->internal;
  const char *msg =
      ((err == MDBX_RESULT_TRUE || err == MDBX_EINTR) && chk->cb->check_break && chk->cb->check_break(chk->usr))
          ? "user break"
          : mdbx_strerror(err);
  if (line)
    chk_line_end(chk_flush(chk_print(line, "%s() failed, error %s (%d)", subj, msg, err)));
  else
    debug_log(MDBX_LOG_ERROR, "mdbx_env_chk", 0, "%s() failed, error %s (%d)", subj, msg, err);
  return err;
}

__cold static void MDBX_PRINTF_ARGS(5, 6)
    chk_object_issue(MDBX_chk_scope_t *const scope, const char *object, uint64_t entry_number, const char *caption,
                     const char *extra_fmt, ...) {
  MDBX_chk_internal_t *const chk = scope->internal;
  MDBX_chk_issue_t *issue = chk->usr->scope->issues;
  while (issue) {
    if (issue->caption == caption) {
      issue->count += 1;
      break;
    } else
      issue = issue->next;
  }
  const bool fresh = issue == nullptr;
  if (fresh) {
    issue = osal_malloc(sizeof(*issue));
    if (likely(issue)) {
      issue->caption = caption;
      issue->count = 1;
      issue->next = chk->usr->scope->issues;
      chk->usr->scope->issues = issue;
    } else
      chk_error_rc(scope, MDBX_ENOMEM, "adding issue");
  }

  va_list args;
  va_start(args, extra_fmt);
  if (chk->cb->issue) {
    mdbx_env_chk_encount_problem(chk->usr);
    chk->cb->issue(chk->usr, object, entry_number, caption, extra_fmt, args);
  } else {
    MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_error);
    if (entry_number != UINT64_MAX)
      chk_print(line, "%s #%" PRIu64 ": %s", object, entry_number, caption);
    else
      chk_print(line, "%s: %s", object, caption);
    if (extra_fmt)
      chk_puts(chk_print_va(chk_puts(line, " ("), extra_fmt, args), ")");
    chk_line_end(fresh ? chk_flush(line) : line);
  }
  va_end(args);
}

__cold static void MDBX_PRINTF_ARGS(2, 3) chk_scope_issue(MDBX_chk_scope_t *const scope, const char *fmt, ...) {
  MDBX_chk_internal_t *const chk = scope->internal;
  va_list args;
  va_start(args, fmt);
  if (likely(chk->cb->issue)) {
    mdbx_env_chk_encount_problem(chk->usr);
    chk->cb->issue(chk->usr, nullptr, 0, nullptr, fmt, args);
  } else
    chk_line_end(chk_print_va(chk_line_begin(scope, MDBX_chk_error), fmt, args));
  va_end(args);
}

__cold static int chk_scope_end(MDBX_chk_internal_t *chk, int err) {
  ASSERT(chk->scope_depth > 0);
  MDBX_chk_scope_t *const inner = chk->scope_stack + chk->scope_depth;
  MDBX_chk_scope_t *const outer = chk->scope_depth ? inner - 1 : nullptr;
  if (!outer || outer->stage != inner->stage) {
    if (err == MDBX_SUCCESS && *chk->problem_counter)
      err = MDBX_PROBLEM;
    else if (*chk->problem_counter == 0 && MDBX_IS_ERROR(err))
      *chk->problem_counter = 1;
    if (chk->problem_counter != &chk->usr->result.total_problems) {
      chk->usr->result.total_problems += *chk->problem_counter;
      chk->problem_counter = &chk->usr->result.total_problems;
    }
    if (chk->cb->stage_end)
      err = chk->cb->stage_end(chk->usr, inner->stage, err);
  }
  if (chk->cb->scope_conclude)
    err = chk->cb->scope_conclude(chk->usr, outer, inner, err);
  chk->usr->scope = outer;
  chk->usr->scope_nesting = chk->scope_depth -= 1;
  if (outer)
    outer->subtotal_issues += inner->subtotal_issues;
  if (chk->cb->scope_pop)
    chk->cb->scope_pop(chk->usr, outer, inner);

  while (inner->issues) {
    MDBX_chk_issue_t *next = inner->issues->next;
    osal_free(inner->issues);
    inner->issues = next;
  }
  memset(inner, -1, sizeof(*inner));
  return err;
}

__cold static int chk_scope_begin_args(MDBX_chk_internal_t *chk, int verbosity_adjustment, enum MDBX_chk_stage stage,
                                       const void *object, size_t *problems, const char *fmt, va_list args) {
  if (unlikely(chk->scope_depth + 1u >= ARRAY_LENGTH(chk->scope_stack)))
    return MDBX_BACKLOG_DEPLETED;

  MDBX_chk_scope_t *const outer = chk->scope_stack + chk->scope_depth;
  const int verbosity = outer->verbosity + (verbosity_adjustment - 1) * (1 << MDBX_chk_severity_prio_shift);
  MDBX_chk_scope_t *const inner = outer + 1;
  memset(inner, 0, sizeof(*inner));
  inner->internal = outer->internal;
  inner->stage = stage ? stage : (stage = outer->stage);
  inner->object = object;
  inner->verbosity = (verbosity < MDBX_chk_warning) ? MDBX_chk_warning : (enum MDBX_chk_severity)verbosity;
  if (problems)
    chk->problem_counter = problems;
  else if (!chk->problem_counter || outer->stage != stage)
    chk->problem_counter = &chk->usr->result.total_problems;

  if (chk->cb->scope_push) {
    const int err = chk->cb->scope_push(chk->usr, outer, inner, fmt, args);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }
  chk->usr->scope = inner;
  chk->usr->scope_nesting = chk->scope_depth += 1;

  if (stage != outer->stage && chk->cb->stage_begin) {
    int err = chk->cb->stage_begin(chk->usr, stage);
    if (unlikely(err != MDBX_SUCCESS)) {
      err = chk_scope_end(chk, err);
      ASSERT(err != MDBX_SUCCESS);
      return err ? err : MDBX_RESULT_TRUE;
    }
  }
  return MDBX_SUCCESS;
}

__cold static int MDBX_PRINTF_ARGS(6, 7)
    chk_scope_begin(MDBX_chk_internal_t *chk, int verbosity_adjustment, enum MDBX_chk_stage stage, const void *object,
                    size_t *problems, const char *fmt, ...) {
  va_list args;
  va_start(args, fmt);
  int rc = chk_scope_begin_args(chk, verbosity_adjustment, stage, object, problems, fmt, args);
  va_end(args);
  return rc;
}

__cold static int chk_scope_restore(MDBX_chk_scope_t *const target, int err) {
  MDBX_chk_internal_t *const chk = target->internal;
  ASSERT(target <= chk->usr->scope);
  while (chk->usr->scope > target)
    err = chk_scope_end(chk, err);
  return err;
}

__cold void chk_scope_pop(MDBX_chk_scope_t *const inner) {
  if (inner && inner > inner->internal->scope_stack)
    chk_scope_restore(inner - 1, MDBX_SUCCESS);
}

__cold static MDBX_chk_scope_t *MDBX_PRINTF_ARGS(3, 4)
    chk_scope_push(MDBX_chk_scope_t *const scope, int verbosity_adjustment, const char *fmt, ...) {
  chk_scope_restore(scope, MDBX_SUCCESS);
  va_list args;
  va_start(args, fmt);
  int err = chk_scope_begin_args(scope->internal, verbosity_adjustment, scope->stage, nullptr, nullptr, fmt, args);
  va_end(args);
  return err ? nullptr : scope + 1;
}

__cold static const char *chk_v2a(MDBX_chk_internal_t *chk, const MDBX_val *val) {
  if (val == MDBX_CHK_MAIN)
    return "@MAIN";
  if (val == MDBX_CHK_GC)
    return "@GC";
  if (val == MDBX_CHK_META)
    return "@META";

  const unsigned char *const data = val->iov_base;
  const size_t len = val->iov_len;
  if (data == MDBX_CHK_MAIN)
    return "@MAIN";
  if (data == MDBX_CHK_GC)
    return "@GC";
  if (data == MDBX_CHK_META)
    return "@META";

  if (!len)
    return "<zero-length>";
  if (!data)
    return "<nullptr>";
  if (len > 65536) {
    const size_t enough = 42;
    if (chk->v2a_buf.iov_len < enough) {
      void *ptr = osal_realloc(chk->v2a_buf.iov_base, enough);
      if (unlikely(!ptr))
        return "<out-of-memory>";
      chk->v2a_buf.iov_base = ptr;
      chk->v2a_buf.iov_len = enough;
    }
    snprintf(chk->v2a_buf.iov_base, chk->v2a_buf.iov_len, "<too-long.%" PRIuSIZE ">", len);
    return chk->v2a_buf.iov_base;
  }

  bool printable = true;
  bool quoting = false;
  size_t xchars = 0;
  for (size_t i = 0; i < len && printable; ++i) {
    quoting = quoting || !(data[i] == '_' || isalnum(data[i]));
    printable = isprint(data[i]) || (data[i] < ' ' && ++xchars < 4 && len > xchars * 4);
  }

  size_t need = len + 1;
  if (quoting || !printable)
    need += len + /* quotes */ 2 + 2 * /* max xchars */ 4;
  if (need > chk->v2a_buf.iov_len) {
    void *ptr = osal_realloc(chk->v2a_buf.iov_base, need);
    if (unlikely(!ptr))
      return "<out-of-memory>";
    chk->v2a_buf.iov_base = ptr;
    chk->v2a_buf.iov_len = need;
  }

  static const char hex[] = "0123456789abcdef";
  char *w = chk->v2a_buf.iov_base;
  if (!quoting) {
    memcpy(w, data, len);
    w += len;
  } else if (printable) {
    *w++ = '\'';
    for (size_t i = 0; i < len; ++i) {
      if (data[i] < ' ') {
        ASSERT((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 4);
        w[0] = '\\';
        w[1] = 'x';
        w[2] = hex[data[i] >> 4];
        w[3] = hex[data[i] & 15];
        w += 4;
      } else if (strchr("\"'`\\", data[i])) {
        ASSERT((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 2);
        w[0] = '\\';
        w[1] = data[i];
        w += 2;
      } else {
        ASSERT((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 1);
        *w++ = data[i];
      }
    }
    *w++ = '\'';
  } else {
    *w++ = '\\';
    *w++ = 'x';
    for (size_t i = 0; i < len; ++i) {
      ASSERT((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 2);
      w[0] = hex[data[i] >> 4];
      w[1] = hex[data[i] & 15];
      w += 2;
    }
  }
  ASSERT((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w);
  *w = 0;
  return chk->v2a_buf.iov_base;
}

__cold static void chk_dispose(MDBX_chk_internal_t *chk) {
  ASSERT(chk->table[FREE_DBI] == &chk->table_gc);
  ASSERT(chk->table[MAIN_DBI] == &chk->table_main);
  for (size_t i = 0; i < ARRAY_LENGTH(chk->table); ++i) {
    MDBX_chk_table_t *const tbl = chk->table[i];
    if (tbl) {
      chk->table[i] = nullptr;
      if (chk->cb->table_dispose && tbl->cookie) {
        chk->cb->table_dispose(chk->usr, tbl);
        tbl->cookie = nullptr;
      }
      if (tbl != &chk->table_gc && tbl != &chk->table_main)
        osal_free(tbl);
    }
  }
  osal_free(chk->v2a_buf.iov_base);
  osal_free(chk->pagemap);
  chk->usr->internal = nullptr;
  chk->usr->scope = nullptr;
  chk->pagemap = nullptr;
  memset(chk, 0xDD, sizeof(*chk));
  osal_free(chk);
}

//-----------------------------------------------------------------------------

__cold static int chk_get_tbl(MDBX_chk_scope_t *const scope, const walk_tbl_t *in, MDBX_chk_table_t **out) {
  MDBX_chk_internal_t *const chk = scope->internal;
  if (chk->last_lookup && chk->last_lookup->name.iov_base == in->name.iov_base) {
    *out = chk->last_lookup;
    return MDBX_SUCCESS;
  }

  for (size_t i = 0; i < ARRAY_LENGTH(chk->table); ++i) {
    MDBX_chk_table_t *tbl = chk->table[i];
    if (!tbl) {
      tbl = osal_calloc(1, sizeof(MDBX_chk_table_t));
      if (unlikely(!tbl)) {
        *out = nullptr;
        return chk_error_rc(scope, MDBX_ENOMEM, "alloc_table");
      }
      chk->table[i] = tbl;
      tbl->flags = in->internal->flags;
      tbl->id = -1;
      tbl->name = in->name;
    }
    if (tbl->name.iov_base == in->name.iov_base) {
      if (tbl->id < 0) {
        tbl->id = (int)i;
        tbl->cookie =
            chk->cb->table_filter ? chk->cb->table_filter(chk->usr, &tbl->name, tbl->flags) : (void *)(intptr_t)-1;
      }
      *out = (chk->last_lookup = tbl);
      return MDBX_SUCCESS;
    }
  }
  chk_scope_issue(scope, "too many tables > %u", (unsigned)ARRAY_LENGTH(chk->table) - CORE_DBS - /* meta */ 1);
  *out = nullptr;
  return MDBX_PROBLEM;
}

//------------------------------------------------------------------------------

__cold static void chk_verbose_meta(MDBX_chk_scope_t *const scope, const unsigned num) {
  MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_verbose);
  MDBX_chk_internal_t *const chk = scope->internal;
  if (line) {
    MDBX_env *const env = chk->usr->env;
    const bool have_bootid = (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) != 0;
    const bool bootid_match = have_bootid && memcmp(&chk->envinfo.mi_bootid.meta[num], &chk->envinfo.mi_bootid.current,
                                                    sizeof(chk->envinfo.mi_bootid.current)) == 0;

    const char *status = "stay";
    if (num == chk->troika.recent)
      status = "head";
    else if (num == TROIKA_TAIL(&chk->troika))
      status = "tail";
    line = chk_print(line, "meta-%u: %s, ", num, status);

    switch (chk->envinfo.mi_meta_sign[num]) {
    case DATASIGN_NONE:
      line = chk_puts(line, "no-sync/legacy");
      break;
    case DATASIGN_WEAK:
      line = chk_print(line, "weak-%s",
                       have_bootid ? (bootid_match ? "intact (same boot-id)" : "dead") : "unknown (no boot-id)");
      break;
    default:
      line = chk_puts(line, "steady");
      break;
    }
    const txnid_t meta_txnid = chk->envinfo.mi_meta_txnid[num];
    line = chk_print(line, " txn#%" PRIaTXN ", ", meta_txnid);
    if (chk->envinfo.mi_bootid.meta[num].x | chk->envinfo.mi_bootid.meta[num].y)
      line = chk_print(line, "boot-id %" PRIx64 "-%" PRIx64 " (%s)", chk->envinfo.mi_bootid.meta[num].x,
                       chk->envinfo.mi_bootid.meta[num].y, bootid_match ? "live" : "dissimilar");
    else
      line = chk_puts(line, "no boot-id");

    if (env->stuck_meta >= 0) {
      if (num == (unsigned)env->stuck_meta)
        line = chk_print(line, ", %s", "forced for checking");
    } else if (meta_txnid > chk->envinfo.mi_recent_txnid &&
               (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE)
      line = chk_print(line, ", rolled-back %" PRIu64 " commit(s) (%" PRIu64 " >>> %" PRIu64 ")",
                       meta_txnid - chk->envinfo.mi_recent_txnid, meta_txnid, chk->envinfo.mi_recent_txnid);
    chk_line_end(line);
  }
}

__cold static int chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, const unsigned deep,
                                const walk_tbl_t *tbl_info, const size_t page_size, const page_type_t pagetype,
                                const txnid_t page_txnid, const MDBX_error_t page_err, const size_t nentries,
                                const size_t payload_bytes, const size_t header_bytes, const size_t unused_bytes,
                                const size_t parent_pgno) {
  MDBX_chk_scope_t *const scope = ctx;
  MDBX_chk_internal_t *const chk = scope->internal;
  MDBX_chk_context_t *const usr = chk->usr;
  MDBX_env *const env = usr->env;
  MDBX_txn *const txn = usr->txn;

  MDBX_chk_table_t *tbl;
  int err = chk_get_tbl(scope, tbl_info, &tbl);
  if (unlikely(err))
    return err;

  if (deep > 42) {
    chk_scope_issue(scope, "too deeply %u, page %zu, parent %zu", deep, pgno, parent_pgno);
    return MDBX_CORRUPTED /* avoid infinite loop/recursion */;
  }
  if (pagetype != page_large)
    histogram_acc(deep, &tbl->histogram.height);
  usr->result.processed_pages += npages;
  const size_t page_bytes = payload_bytes + header_bytes + unused_bytes;

  if (pagetype < page_sub_leaf) {
    const txnid_t basis = txn_basis_snapshot(txn);
    const txnid_t age = (page_txnid >= MIN_TXNID)
                            ? ((page_txnid > basis) ? /* dirty */ 0 : (basis - page_txnid) / xMDBX_TXNID_STEP + 1)
                            : SIZE_MAX;
    histogram_acc_ex((age < SIZE_MAX) ? age : SIZE_MAX - 1, &usr->result.histogram_page_age, HISTOGRAM_LE0);
    histogram_acc_ex((age < SIZE_MAX) ? age : SIZE_MAX - 1, &tbl->histogram.page_age, HISTOGRAM_LE0);

    histogram_acc(pgno, &tbl->histogram.pgno);
    if (tbl->id != FREE_DBI)
      histogram_acc(pgno, &usr->result.histogram_pgno_payload);
  }

  int height = deep + 1;
  if (tbl->id >= CORE_DBS)
    height -= usr->txn->dbs[MAIN_DBI].height;
  const tree_t *nested = tbl_info->nested;
  if (nested) {
    if (tbl->flags & MDBX_DUPSORT)
      height -= tbl_info->internal->height;
    else {
      chk_object_issue(scope, "nested tree", pgno, "unexpected", "table %s flags 0x%x, deep %i",
                       chk_v2a(chk, &tbl->name), tbl->flags, deep);
      nested = nullptr;
    }
  }

  const char *pagetype_caption;
  bool branch = false;
  struct MDBX_chk_histogram *density = nullptr;
  switch (pagetype) {
  default:
    chk_object_issue(scope, "page", pgno, "unknown page-type", "type %u, deep %i, parent %zu", (unsigned)pagetype, deep,
                     parent_pgno);
    pagetype_caption = "unknown";
    tbl->pages.broken += npages;
    break;
  case page_broken:
    ASSERT(page_err != MDBX_SUCCESS);
    pagetype_caption = "broken";
    tbl->pages.broken += npages;
    break;
  case page_sub_broken:
    ASSERT(page_err != MDBX_SUCCESS);
    pagetype_caption = "broken-subpage";
    tbl->pages.broken += npages;
    break;
  case page_large:
    pagetype_caption = "large";
    histogram_acc(npages, &tbl->histogram.large_pages);
    density = &tbl->histogram.large_or_nested_density;
    if (tbl->flags & MDBX_DUPSORT)
      chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i, parent %zu",
                       (unsigned)pagetype, chk_v2a(chk, &tbl->name), tbl->flags, deep, parent_pgno);
    break;
  case page_branch:
    branch = true;
    if (!nested) {
      pagetype_caption = "branch";
      tbl->pages.branch += 1;
      density = &tbl->histogram.tree_density;
    } else {
      pagetype_caption = "nested-branch";
      tbl->pages.nested_branch += 1;
      density = &tbl->histogram.large_or_nested_density;
    }
    break;
  case page_dupfix_leaf:
    if (!nested)
      chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i, parent %zu",
                       (unsigned)pagetype, chk_v2a(chk, &tbl->name), tbl->flags, deep, parent_pgno);
    /* fall through */
    __fallthrough;
  case page_leaf:
    if (!nested) {
      pagetype_caption = "leaf";
      tbl->pages.leaf += 1;
      density = &tbl->histogram.tree_density;
      if (height != tbl_info->internal->height)
        chk_object_issue(scope, "page", pgno, "wrong tree height", "actual %i != %i table %s, parent %zu", height,
                         tbl_info->internal->height, chk_v2a(chk, &tbl->name), parent_pgno);
    } else {
      pagetype_caption = (pagetype == page_leaf) ? "nested-leaf" : "nested-leaf-dupfix";
      tbl->pages.nested_leaf += 1;
      density = &tbl->histogram.large_or_nested_density;
      if (chk->last_nested_root != nested->root) {
        histogram_acc(height, &tbl->histogram.nested_height_or_gc_span_length);
        chk->last_nested_root = nested->root;
      }
      if (height != nested->height)
        chk_object_issue(scope, "page", pgno, "wrong nested-tree height", "actual %i != %i dupsort-node %s, parent %zu",
                         height, nested->height, chk_v2a(chk, &tbl->name), parent_pgno);
    }
    break;
  case page_sub_dupfix_leaf:
  case page_sub_leaf:
    pagetype_caption = (pagetype == page_sub_leaf) ? "subleaf-dupsort" : "subleaf-dupfix";
    tbl->pages.nested_subleaf += 1;
    if ((tbl->flags & MDBX_DUPSORT) == 0 || nested)
      chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i, parent %zu",
                       (unsigned)pagetype, chk_v2a(chk, &tbl->name), tbl->flags, deep, parent_pgno);
    else
      density = &tbl->histogram.large_or_nested_density;
    break;
  }

  if (density)
    histogram_acc((page_size - unused_bytes) * 100 / page_size, density);

  if (npages) {
    if (tbl->cookie) {
      MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_extra);
      if (npages == 1)
        chk_print(line, "%s-page %" PRIuSIZE, pagetype_caption, pgno);
      else
        chk_print(line, "%s-span %" PRIuSIZE "[%u]", pagetype_caption, pgno, npages);
      chk_line_end(chk_print(
          line, " of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR ", unused %" PRIiPTR ", deep %i",
          chk_v2a(chk, &tbl->name), header_bytes, (pagetype == page_branch) ? "keys" : "entries", nentries,
          payload_bytes, unused_bytes, deep));
    }

    bool already_used = false;
    for (unsigned n = 0; n < npages; ++n) {
      const size_t spanpgno = pgno + n;
      if (spanpgno >= usr->result.alloc_pages) {
        chk_object_issue(scope, "page", spanpgno, "wrong page-no", "%s-page: %" PRIuSIZE " > %" PRIuSIZE ", deep %i",
                         pagetype_caption, spanpgno, usr->result.alloc_pages, deep);
        tbl->pages.all += 1;
      } else if (chk->pagemap[spanpgno]) {
        const MDBX_chk_table_t *const rival = chk->table[chk->pagemap[spanpgno] - 1];
        chk_object_issue(scope, "page", spanpgno, (branch && rival == tbl) ? "loop" : "already used",
                         "%s-page: by %s, deep %i, parent %zu", pagetype_caption, chk_v2a(chk, &rival->name), deep,
                         parent_pgno);
        already_used = true;
      } else {
        chk->pagemap[spanpgno] = (int16_t)tbl->id + 1;
        tbl->pages.all += 1;
      }
    }

    if (already_used)
      return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */
                    : MDBX_SUCCESS;
  }

  if (MDBX_IS_ERROR(page_err)) {
    chk_object_issue(scope, "page", pgno, "invalid/corrupted", "%s-page, parent %zu", pagetype_caption, parent_pgno);
  } else {
    if (unused_bytes > page_size)
      chk_object_issue(scope, "page", pgno, "illegal unused-bytes", "%s-page: %u < %" PRIuSIZE " < %u, parent %zu",
                       pagetype_caption, 0, unused_bytes, env->ps, parent_pgno);

    if (header_bytes < (int)sizeof(long) || (size_t)header_bytes >= env->ps - sizeof(long)) {
      chk_object_issue(scope, "page", pgno, "illegal header-length",
                       "%s-page: %" PRIuSIZE " < %" PRIuSIZE " < %" PRIuSIZE ", parent %zu", pagetype_caption,
                       sizeof(long), header_bytes, env->ps - sizeof(long), parent_pgno);
    }
    if (nentries < 1 || (pagetype == page_branch && nentries < 2)) {
      chk_object_issue(scope, "page", pgno, nentries ? "half-empty" : "empty",
                       "%s-page: payload %" PRIuSIZE " bytes, %" PRIuSIZE " entries, deep %i, parent %zu",
                       pagetype_caption, payload_bytes, nentries, deep, parent_pgno);
      tbl->pages.empty += 1;
    }

    if (npages) {
      if (page_bytes != page_size) {
        chk_object_issue(scope, "page", pgno, "misused",
                         "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR "h + %" PRIuPTR "p + %" PRIuPTR
                         "u), deep %i, parent %zu",
                         pagetype_caption, page_size, page_bytes, header_bytes, payload_bytes, unused_bytes, deep,
                         parent_pgno);
        if (page_size > page_bytes)
          tbl->lost_bytes += page_size - page_bytes;
      } else {
        tbl->payload_bytes += payload_bytes + header_bytes;
        usr->result.total_payload_bytes += payload_bytes + header_bytes;
      }
    }
  }
  return chk_check_break(scope);
}

__cold static int chk_tree(MDBX_chk_scope_t *const scope) {
  MDBX_chk_internal_t *const chk = scope->internal;
  MDBX_chk_context_t *const usr = chk->usr;
  MDBX_env *const env = usr->env;
  MDBX_txn *const txn = usr->txn;

#if defined(_WIN32) || defined(_WIN64)
  SetLastError(ERROR_SUCCESS);
#else
  errno = 0;
#endif /* Windows */
  chk->pagemap = osal_calloc(usr->result.alloc_pages, sizeof(*chk->pagemap));
  if (!chk->pagemap) {
    int err = osal_get_errno();
    return chk_error_rc(scope, err ? err : MDBX_ENOMEM, "calloc");
  }

  if (scope->verbosity > MDBX_chk_info)
    chk_scope_push(scope, 0, "Walking pages...");
  /* always skip key ordering checking
   * to avoid MDBX_CORRUPTED in case custom comparators were used */
  usr->result.processed_pages = NUM_METAS;
  int err = walk_pages(txn, chk_pgvisitor, scope, dont_check_keys_ordering);
  if (err != MDBX_SUCCESS) {
    chk_error_rc(scope, err, "walk_pages");
    return err;
  }

  for (size_t n = NUM_METAS; n < usr->result.alloc_pages; ++n)
    if (!chk->pagemap[n])
      usr->result.unused_pages += 1;

  MDBX_chk_table_t total;
  memset(&total, 0, sizeof(total));
  total.pages.all = NUM_METAS;
  for (size_t i = 0; i < ARRAY_LENGTH(chk->table) && chk->table[i]; ++i) {
    MDBX_chk_table_t *const tbl = chk->table[i];
    total.payload_bytes += tbl->payload_bytes;
    total.lost_bytes += tbl->lost_bytes;
    total.pages.all += tbl->pages.all;
    total.pages.empty += tbl->pages.empty;
    total.pages.broken += tbl->pages.broken;
    total.pages.branch += tbl->pages.branch;
    total.pages.leaf += tbl->pages.leaf;
    total.pages.nested_branch += tbl->pages.nested_branch;
    total.pages.nested_leaf += tbl->pages.nested_leaf;
    total.pages.nested_subleaf += tbl->pages.nested_subleaf;
  }
  ASSERT(total.pages.all == usr->result.processed_pages);

  const size_t total_page_bytes = pgno2bytes(env, total.pages.all);
  if (usr->scope->subtotal_issues || usr->scope->verbosity >= MDBX_chk_verbose)
    chk_line_end(chk_print(chk_line_begin(usr->scope, MDBX_chk_resolution),
                           "walked %zu pages, left/unused %zu"
                           ", %" PRIuSIZE " problem(s)",
                           usr->result.processed_pages, usr->result.unused_pages, usr->scope->subtotal_issues));

  err = chk_scope_restore(scope, err);
  if (scope->verbosity > MDBX_chk_info) {
    for (size_t i = 0; i < ARRAY_LENGTH(chk->table) && chk->table[i]; ++i) {
      MDBX_chk_table_t *const tbl = chk->table[i];
      MDBX_chk_scope_t *inner =
          chk_scope_push(scope, 0, (tbl->pages.all ? "b-tree %s, subtotal %" PRIuSIZE " pages:" : "b-tree %s: empty"),
                         chk_v2a(chk, &tbl->name), tbl->pages.all);
      if (tbl->pages.all) {
        MDBX_chk_line_t *line = chk_line_begin(inner, MDBX_chk_info);
        if (line) {
          line = chk_puts(line, "pages composition: ");
          if (tbl->pages.broken)
            line = chk_print(line, "broken %" PRIuSIZE ", ", tbl->pages.broken);
          if (tbl->pages.broken != tbl->pages.all) {
            line = chk_print(line, "branch %" PRIuSIZE ", leaf %" PRIuSIZE, tbl->pages.branch, tbl->pages.leaf);
            if (tbl->pages.nested_subleaf || (tbl->flags & MDBX_DUPSORT) != 0)
              line = chk_print(line, ", subleaf %" PRIuSIZE, tbl->pages.nested_subleaf);
            if (tbl->pages.nested_branch || (tbl->flags & MDBX_DUPSORT) != 0)
              line = chk_print(line, ", nested-branch %" PRIuSIZE, tbl->pages.nested_branch);
            if (tbl->pages.nested_leaf || (tbl->flags & MDBX_DUPSORT) != 0)
              line = chk_print(line, ", nested-leaf %" PRIuSIZE, tbl->pages.nested_leaf);
            if (tbl->histogram.large_pages.count || (tbl->flags & MDBX_DUPSORT) == 0) {
              line = chk_print(line, ", large %" PRIuSIZE, tbl->histogram.large_pages.count);
              if (tbl->histogram.large_pages.amount | tbl->histogram.large_pages.count)
                line = histogram_print(inner, line, &tbl->histogram.large_pages, " amount", "single", true);
            }
            line = histogram_dist(chk_line_feed(line), &tbl->histogram.height, "tree levels", "1", false);
            if ((tbl->flags & MDBX_DUPSORT) != 0 ||
                (tbl->histogram.nested_height_or_gc_span_length.count && tbl != &chk->table_gc)) {
              line = chk_print(chk_line_feed(line),
                               "nested tree(s): quantity %" PRIuSIZE ", subtotal pages %" PRIuSIZE ", ",
                               tbl->histogram.nested_height_or_gc_span_length.count,
                               tbl->pages.nested_branch + tbl->pages.nested_leaf);
              if (tbl != &chk->table_gc)
                line = histogram_dist(line, &tbl->histogram.nested_height_or_gc_span_length, "levels", "1", false);
            }
            line = chk_line_feed(line);

            const size_t bytes = pgno2bytes(env, tbl->pages.all);
            line = chk_print_bytes_percent(line, "pages density: subtotal\0\0", bytes, total_page_bytes);
            line = chk_print_percent(line, ", payload\0\0", tbl->payload_bytes, bytes, "");
            line = chk_print_percent(line, ", unused\0\0", bytes - tbl->payload_bytes, bytes, "");
            if (tbl->pages.empty)
              line = chk_print(line, ", %" PRIuSIZE " empty pages", tbl->pages.empty);
            if (tbl->lost_bytes)
              line = chk_print(line, ", %" PRIuSIZE " bytes lost", tbl->lost_bytes);

            line = histogram_dist(chk_line_feed(line), &tbl->histogram.tree_density, "pages %-density distribution",
                                  "1", false);
            line = histogram_dist(chk_line_feed(line), &tbl->histogram.large_or_nested_density,
                                  (tbl->flags & MDBX_DUPSORT) ? "nested %-density distribution"
                                                              : "large pages %-density distribution",
                                  "1", false);
          }
          histogram_dist(chk_line_feed(line), &tbl->histogram.page_age, "pages age distribution", "dirty", false);
          chk_line_end(line);
        }
        if (tbl->histogram.pgno.count) {
          line = chk_line_begin(inner, (tbl->id != FREE_DBI) ? MDBX_chk_verbose : MDBX_chk_info);
          histogram_dist(line, &tbl->histogram.pgno, "pgno distribution", "bad", false);
          chk_line_end(line);
        }
      }
      chk_scope_restore(scope, 0);
    }
  }

  MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_resolution);
  line = chk_print(line, "summary: total %" PRIuSIZE " bytes", total_page_bytes);
  line =
      chk_print_percent(line, ", payload\0 average density\0", usr->result.total_payload_bytes, total_page_bytes, "");
  line = chk_print_percent(line, ", unused\0 average sparsity\0", total_page_bytes - usr->result.total_payload_bytes,
                           total_page_bytes, "");
  if (total.pages.empty)
    line = chk_print(line, ", %" PRIuSIZE " empty pages", total.pages.empty);
  if (total.lost_bytes)
    line = chk_print(line, ", %" PRIuSIZE " bytes lost", total.lost_bytes);
  chk_line_end(line);
  return err;
}

typedef int(chk_kv_visitor)(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number,
                            const MDBX_val *key, const MDBX_val *data);

__cold static int chk_handle_kv(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number,
                                const MDBX_val *key, const MDBX_val *data) {
  MDBX_chk_internal_t *const chk = scope->internal;
  int err = MDBX_SUCCESS;
  ASSERT(tbl->cookie);
  if (chk->cb->table_handle_kv)
    err = chk->cb->table_handle_kv(chk->usr, tbl, record_number, key, data);
  return err ? err : chk_check_break(scope);
}

__cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, MDBX_chk_table_t *tbl, chk_kv_visitor *handler) {
  MDBX_chk_internal_t *const chk = scope->internal;
  MDBX_chk_context_t *const usr = chk->usr;
  MDBX_env *const env = usr->env;
  MDBX_txn *const txn = usr->txn;
  MDBX_cursor *cursor = nullptr;
  size_t record_count = 0, dups = 0, sub_databases = 0;
  int err;

  if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & txn->flags) {
    chk_line_end(chk_flush(chk_print(chk_line_begin(scope, MDBX_chk_error),
                                     "abort processing %s due to a previous error", chk_v2a(chk, &tbl->name))));
    err = MDBX_BAD_TXN;
    goto bailout;
  }

  if (0 > (int)dbi) {
    err = dbi_open(txn, &tbl->name, MDBX_DB_ACCEDE, &dbi,
                   (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr,
                   (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr);
    if (unlikely(err)) {
      cASSERT0(txn, dbi >= txn->env->n_dbi || (txn->env->dbs_flags[dbi] & DB_VALID) == 0);
      chk_error_rc(scope, err, "mdbx_dbi_open");
      goto bailout;
    }
    cASSERT0(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0);
  }

  const tree_t *const db = txn->dbs + dbi;
  if (handler) {
    const char *key_mode = nullptr;
    switch (tbl->flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) {
    case 0:
      key_mode = "usual";
      break;
    case MDBX_REVERSEKEY:
      key_mode = "reserve";
      break;
    case MDBX_INTEGERKEY:
      key_mode = "ordinal";
      break;
    case MDBX_REVERSEKEY | MDBX_INTEGERKEY:
      key_mode = "msgpack";
      break;
    default:
      key_mode = "inconsistent";
      chk_scope_issue(scope, "wrong key-mode (0x%x)", tbl->flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY));
    }

    const char *value_mode = nullptr;
    switch (tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP)) {
    case 0:
      value_mode = "single";
      break;
    case MDBX_DUPSORT:
      value_mode = "multi";
      break;
    case MDBX_DUPSORT | MDBX_REVERSEDUP:
      value_mode = "multi-reverse";
      break;
    case MDBX_DUPSORT | MDBX_DUPFIXED:
      value_mode = "multi-samelength";
      break;
    case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP:
      value_mode = "multi-reverse-samelength";
      break;
    case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP:
      value_mode = "multi-ordinal";
      break;
    case MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
      value_mode = "multi-msgpack";
      break;
    case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
      value_mode = "reserved";
      break;
    default:
      value_mode = "inconsistent";
      chk_scope_issue(scope, "wrong value-mode (0x%x)",
                      tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP));
    }

    MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_info);
    line = chk_print(line, "key-value kind: %s-key => %s-value", key_mode, value_mode);
    line = chk_print(line, ", flags:");
    if (!tbl->flags)
      line = chk_print(line, " none");
    else {
      const uint8_t f[] = {
          MDBX_DUPSORT, MDBX_INTEGERKEY, MDBX_REVERSEKEY, MDBX_DUPFIXED, MDBX_REVERSEDUP, MDBX_INTEGERDUP, 0};
      const char *const t[] = {"dupsort", "integerkey", "reversekey", "dupfix", "reversedup", "integerdup"};
      for (size_t i = 0; f[i]; i++)
        if (tbl->flags & f[i])
          line = chk_print(line, " %s", t[i]);
    }
    chk_line_end(chk_print(line, " (0x%02X)", tbl->flags));

    line = chk_print(chk_line_begin(scope, MDBX_chk_verbose), "entries %" PRIu64 ", sequence %" PRIu64, db->items,
                     db->sequence);
    if (db->mod_txnid)
      line = chk_print(line, ", last modification txn#%" PRIaTXN, db->mod_txnid);
    if (db->root != P_INVALID)
      line = chk_print(line, ", root #%" PRIaPGNO, db->root);
    chk_line_end(line);
    chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_verbose),
                           "b-tree depth %u, pages: branch %" PRIaPGNO ", leaf %" PRIaPGNO ", large %" PRIaPGNO,
                           db->height, db->branch_pages, db->leaf_pages, db->large_pages));

    if ((chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) == 0) {
      const size_t branch_pages = tbl->pages.branch + tbl->pages.nested_branch;
      const size_t leaf_pages = tbl->pages.leaf + tbl->pages.nested_leaf;
      const size_t subtotal_pages = db->branch_pages + db->leaf_pages + db->large_pages;
      if (subtotal_pages != tbl->pages.all)
        chk_scope_issue(scope, "%s pages mismatch (%" PRIuSIZE " != walked %" PRIuSIZE ")", "subtotal", subtotal_pages,
                        tbl->pages.all);
      if (db->branch_pages != branch_pages)
        chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "branch", db->branch_pages,
                        branch_pages);
      if (db->leaf_pages != leaf_pages)
        chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "all-leaf", db->leaf_pages,
                        leaf_pages);
      if (db->large_pages != tbl->histogram.large_pages.amount)
        chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "large/overlow",
                        db->large_pages, tbl->histogram.large_pages.amount);
    }
  }

  err = mdbx_cursor_open(txn, dbi, &cursor);
  if (unlikely(err)) {
    chk_error_rc(scope, err, "mdbx_cursor_open");
    goto bailout;
  }
  if (chk->flags & MDBX_CHK_IGNORE_ORDER) {
    cursor->checking |= z_ignord | z_pagecheck;
    if (cursor->subcur)
      cursor->subcur->cursor.checking |= z_ignord | z_pagecheck;
  }

  const size_t maxkeysize = mdbx_env_get_maxkeysize_ex(env, tbl->flags);
  MDBX_val prev_key = {nullptr, 0}, prev_data = {nullptr, 0};
  MDBX_val key, data;
  size_t dups_count = 0;
  err = mdbx_cursor_get(cursor, &key, &data, MDBX_FIRST);
  while (err == MDBX_SUCCESS) {
    err = chk_check_break(scope);
    if (unlikely(err))
      goto bailout;

    bool bad_key = false;
    if (key.iov_len > maxkeysize) {
      chk_object_issue(scope, "entry", record_count, "key length exceeds max-key-size", "%" PRIuPTR " > %" PRIuPTR,
                       key.iov_len, maxkeysize);
      bad_key = true;
    } else if ((tbl->flags & MDBX_INTEGERKEY) && key.iov_len != 8 && key.iov_len != 4) {
      chk_object_issue(scope, "entry", record_count, "wrong key length", "%" PRIuPTR " != 4or8", key.iov_len);
      bad_key = true;
    }

    bool bad_data = false;
    if ((tbl->flags & MDBX_INTEGERDUP) && data.iov_len != 8 && data.iov_len != 4) {
      chk_object_issue(scope, "entry", record_count, "wrong data length", "%" PRIuPTR " != 4or8", data.iov_len);
      bad_data = true;
    }

    if (prev_key.iov_base) {
      if (key.iov_base == prev_key.iov_base)
        dups_count += 1;
      else {
        histogram_acc(dups_count, &tbl->histogram.multival);
        dups_count = 0;
      }
      if (prev_data.iov_base && !bad_data && (tbl->flags & MDBX_DUPFIXED) && prev_data.iov_len != data.iov_len) {
        chk_object_issue(scope, "entry", record_count, "different data length", "%" PRIuPTR " != %" PRIuPTR,
                         prev_data.iov_len, data.iov_len);
        bad_data = true;
      }

      if (!bad_key) {
        int cmp = mdbx_cmp(txn, dbi, &key, &prev_key);
        if (cmp == 0) {
          ++dups;
          if ((tbl->flags & MDBX_DUPSORT) == 0) {
            chk_object_issue(scope, "entry", record_count, "duplicated entries", nullptr);
            if (prev_data.iov_base && data.iov_len == prev_data.iov_len &&
                memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0)
              chk_object_issue(scope, "entry", record_count, "complete duplicate", nullptr);
          } else if (!bad_data && prev_data.iov_base) {
            cmp = mdbx_dcmp(txn, dbi, &data, &prev_data);
            if (cmp == 0)
              chk_object_issue(scope, "entry", record_count, "complete duplicate", nullptr);
            else if (cmp < 0 && !(chk->flags & MDBX_CHK_IGNORE_ORDER))
              chk_object_issue(scope, "entry", record_count, "wrong order of multi-values", nullptr);
          }
        } else if (cmp < 0 && !(chk->flags & MDBX_CHK_IGNORE_ORDER))
          chk_object_issue(scope, "entry", record_count, "wrong order of entries", nullptr);
      }
    }

    if (!bad_key) {
      if (!prev_key.iov_base && (tbl->flags & MDBX_INTEGERKEY))
        chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "fixed key-size %" PRIuSIZE, key.iov_len));
      prev_key = key;
    }
    if (!bad_data) {
      if (!prev_data.iov_base && (tbl->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)))
        chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "fixed data-size %" PRIuSIZE, data.iov_len));
      prev_data = data;
    }

    record_count++;
    histogram_acc(key.iov_len, &tbl->histogram.key_len);
    histogram_acc(data.iov_len, &tbl->histogram.val_len);

    const node_t *const node = page_node(cursor->pg[cursor->top], cursor->ki[cursor->top]);
    if (node_flags(node) == N_TREE) {
      if (dbi != MAIN_DBI || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)))
        chk_object_issue(scope, "entry", record_count, "unexpected table", "node-flags 0x%x", node_flags(node));
      else if (data.iov_len != sizeof(tree_t))
        chk_object_issue(scope, "entry", record_count, "wrong table node size", "node-size %" PRIuSIZE " != %" PRIuSIZE,
                         data.iov_len, sizeof(tree_t));
      else if (scope->stage == MDBX_chk_maindb)
        /* подсчитываем table при первом проходе */
        sub_databases += 1;
      else {
        /* обработка table при втором проходе */
        tree_t aligned_db;
        memcpy(&aligned_db, data.iov_base, sizeof(aligned_db));
        walk_tbl_t tbl_info = {.name = key};
        tbl_info.internal = &aligned_db;
        MDBX_chk_table_t *table;
        err = chk_get_tbl(scope, &tbl_info, &table);
        if (unlikely(err))
          goto bailout;
        if (table->cookie) {
          err = chk_scope_begin(chk, 0, MDBX_chk_tables, table, &usr->result.problems_kv, "Processing table %s...",
                                chk_v2a(chk, &table->name));
          if (likely(!err)) {
            err = chk_db(usr->scope, (MDBX_dbi)-1, table, chk_handle_kv);
            if (err != MDBX_EINTR && err != MDBX_RESULT_TRUE)
              usr->result.table_processed += 1;
          }
          err = chk_scope_restore(scope, err);
          if (unlikely(err))
            goto bailout;
        } else
          chk_line_end(chk_flush(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s...",
                                           chk_v2a(chk, &table->name))));
      }
    } else if (handler) {
      err = handler(scope, tbl, record_count, &key, &data);
      if (unlikely(err))
        goto bailout;
    }

    err = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT);
  }

  if (prev_key.iov_base)
    histogram_acc(dups_count, &tbl->histogram.multival);

  err = (err != MDBX_NOTFOUND) ? chk_error_rc(scope, err, "mdbx_cursor_get") : MDBX_SUCCESS;
  if (err == MDBX_SUCCESS && record_count != db->items)
    chk_scope_issue(scope, "different number of entries %" PRIuSIZE " != %" PRIu64, record_count, db->items);
bailout:
  if (cursor) {
    if (handler) {
      if (record_count) {
        MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_info);
        line = histogram_dist(line, &tbl->histogram.key_len, "key length distribution", "0/1", false);
        chk_line_feed(line);
        line = histogram_dist(line, &tbl->histogram.val_len, "value length distribution", "0/1", false);
        if (tbl->histogram.multival.amount) {
          chk_line_feed(line);
          line = histogram_dist(line, &tbl->histogram.multival, "number of multi-values distribution", "single", false);
          chk_line_feed(line);
          line =
              chk_print(line, "number of keys %" PRIuSIZE ", average values per key ", tbl->histogram.multival.count);
          line = chk_print_ratio(line, record_count, tbl->histogram.multival.count, 1);
        }
        chk_line_end(line);
      }
      if (scope->stage == MDBX_chk_maindb)
        usr->result.table_total = sub_databases;
      if (chk->cb->table_conclude)
        err = chk->cb->table_conclude(usr, tbl, cursor, err);
      MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_resolution);
      if (record_count | tbl->histogram.key_len.amount | tbl->histogram.val_len.amount | scope->subtotal_issues) {
        line = chk_print(line, "summary: %" PRIuSIZE " records,", record_count);
        if (dups || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)))
          line = chk_print(line, " %" PRIuSIZE " dups,", dups);
        if (sub_databases || dbi == MAIN_DBI)
          line = chk_print(line, " %" PRIuSIZE " tables,", sub_databases);
        line = chk_print(line,
                         " %" PRIuSIZE " key's bytes,"
                         " %" PRIuSIZE " data's bytes,"
                         " %" PRIuSIZE " problem(s)",
                         tbl->histogram.key_len.amount, tbl->histogram.val_len.amount, scope->subtotal_issues);
      } else {
        line = chk_puts(line, "empty");
      }
      chk_line_end(chk_flush(line));
    }

    mdbx_cursor_close(cursor);
    if (!txn->cursors[dbi] && (txn->dbi_state[dbi] & DBI_FRESH))
      mdbx_dbi_close(env, dbi);
  }

  return err;
}

__cold static int chk_handle_gc(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number,
                                const MDBX_val *key, const MDBX_val *data) {
  MDBX_chk_internal_t *const chk = scope->internal;
  MDBX_chk_context_t *const usr = chk->usr;
  ASSERT(tbl == &chk->table_gc);
  (void)tbl;
  const char *bad = "";

  if (key->iov_len != sizeof(txnid_t))
    chk_object_issue(scope, "entry", record_number, "wrong txn-id size", "key-size %" PRIuSIZE, key->iov_len);
  else {
    const txnid_t txnid = unaligned_peek_u64(4, key->iov_base);
    if (txnid < 1 || txnid > usr->txn->txnid)
      chk_object_issue(scope, "entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid);
    else {
      const const_pnl_t pnl = data->iov_base;
      if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t))
        chk_object_issue(scope, "entry", txnid, "wrong pnl size", "%" PRIuPTR, data->iov_len);
      if ((size_t)data->iov_base % sizeof(pgno_t))
        chk_object_issue(scope, "entry", txnid, "invalid pnl alignment", "%" PRIuPTR, (size_t)data->iov_base);
      size_t number = (data->iov_len >= sizeof(pgno_t) && (size_t)data->iov_base % sizeof(pgno_t) == 0) ? *pnl : 0;
      if (number > PAGELIST_LIMIT)
        chk_object_issue(scope, "entry", txnid, "wrong idl length", "%" PRIuPTR, number);
      else if ((number + 1) * sizeof(pgno_t) > data->iov_len) {
        chk_object_issue(scope, "entry", txnid, "trimmed idl", "%" PRIuSIZE " > %" PRIuSIZE " (corruption)",
                         (number + 1) * sizeof(pgno_t), data->iov_len);
        number = data->iov_len / sizeof(pgno_t) - 1;
      } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >=
                 /* LY: allow gap up to two page. it is ok
                  * and better than shrink-and-retry inside gc_update() */
                 usr->env->ps * 2)
        chk_object_issue(scope, "entry", txnid, "extra idl space",
                         "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", (number + 1) * sizeof(pgno_t),
                         data->iov_len);

      usr->result.gc_pages += number;
      if (chk->envinfo.mi_latter_reader_txnid >= txnid)
        usr->result.reclaimable_pages += number;

      size_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : usr->txn->geo.first_unallocated;
      for (size_t i = 1; i <= number; ++i) {
        const size_t pgno = pnl[i];
        histogram_acc(pgno, (chk->envinfo.mi_latter_reader_txnid > txnid)
                                ? /* account reclaimable as GC-related*/ &tbl->histogram.pgno
                                : &usr->result.histogram_pgno_retained);

        if (pgno < NUM_METAS)
          chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " < meta-pages %u", pgno,
                           NUM_METAS);
        else if (pgno >= usr->result.backed_pages)
          chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " > backed-pages %" PRIuSIZE,
                           pgno, usr->result.backed_pages);
        else if (pgno >= usr->result.alloc_pages)
          chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " > alloc-pages %" PRIuSIZE,
                           pgno, usr->result.alloc_pages - 1);
        else {
          if (unlikely(MDBX_PNL_DISORDERED(prev, pgno))) {
            bad = " [bad sequence]";
            chk_object_issue(scope, "entry", txnid, "bad sequence", "%" PRIuSIZE " %c [%" PRIuSIZE "].%" PRIuSIZE, prev,
                             (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), i, pgno);
          }
          if (chk->pagemap) {
            const intptr_t id = chk->pagemap[pgno];
            if (id == 0)
              chk->pagemap[pgno] = -1 /* mark the pgno listed in GC */;
            else if (id > 0) {
              ASSERT(id - 1 <= (intptr_t)ARRAY_LENGTH(chk->table));
              chk_object_issue(scope, "page", pgno, "already used", "by %s", chk_v2a(chk, &chk->table[id - 1]->name));
            } else
              chk_object_issue(scope, "page", pgno, "already listed in GC", nullptr);
          }
        }
        prev = pgno;
      }
      if (tbl->cookie) {
        chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_details),
                               "transaction %" PRIaTXN ", %" PRIuSIZE " pages, maxspan %" PRIuSIZE "%s", txnid, number,
                               pnl_maxspan(pnl), bad));
        for (size_t span, i = 1; i <= number; i += span) {
          span = pnl_scan_span(pnl, i);
          const size_t pgno = pnl[i];
          histogram_acc(span, &tbl->histogram.nested_height_or_gc_span_length);
          MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_extra);
          if (line) {
            if (span > 1)
              line = chk_print(line, "%9" PRIuSIZE "[%" PRIuSIZE "]", pgno, span);
            else
              line = chk_print(line, "%9" PRIuSIZE, pgno);
            chk_line_end(line);
            int err = chk_check_break(scope);
            if (err)
              return err;
          }
        }
      }
    }
  }
  return chk_check_break(scope);
}

__cold static int env_chk(MDBX_chk_scope_t *const scope) {
  MDBX_chk_internal_t *const chk = scope->internal;
  MDBX_chk_context_t *const usr = chk->usr;
  MDBX_env *const env = usr->env;
  MDBX_txn *const txn = usr->txn;
  int err = env_info(env, txn, &chk->envinfo, &chk->troika);
  if (unlikely(err))
    return chk_error_rc(scope, err, "env_info");

  MDBX_chk_line_t *line =
      chk_puts(chk_line_begin(scope, MDBX_chk_info - (1 << MDBX_chk_severity_prio_shift)), "dxb-id ");
  if (chk->envinfo.mi_dxbid.x | chk->envinfo.mi_dxbid.y)
    line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, chk->envinfo.mi_dxbid.x, chk->envinfo.mi_dxbid.y);
  else
    line = chk_puts(line, "is absent");
  chk_line_end(line);

  line = chk_puts(chk_line_begin(scope, MDBX_chk_info), "current boot-id ");
  if (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y)
    line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, chk->envinfo.mi_bootid.current.x,
                     chk->envinfo.mi_bootid.current.y);
  else
    line = chk_puts(line, "is unavailable");
  chk_line_end(line);

  line = chk_print_size(chk_line_begin(scope, MDBX_chk_verbose), "system unified page cache block ",
                        chk->envinfo.mi_sys_upcblk, "");
  chk_line_end(line);

  env->dxb_mmap.filesize = chk->envinfo.mi_dxb_fsize;

  //--------------------------------------------------------------------------

  err = chk_scope_begin(chk, 1, MDBX_chk_meta, nullptr, &usr->result.problems_meta, "Peek the meta-pages...");
  if (likely(!err)) {
    MDBX_chk_scope_t *const inner = usr->scope;
    const uint64_t dxbfile_pages = env->dxb_mmap.filesize >> env->ps2ln;
    usr->result.alloc_pages = txn->geo.first_unallocated;
    usr->result.backed_pages = bytes2pgno(env, env->dxb_mmap.current);
    if (unlikely(usr->result.backed_pages > dxbfile_pages))
      chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, usr->result.backed_pages, dxbfile_pages);
    if (unlikely(dxbfile_pages < NUM_METAS))
      chk_scope_issue(inner, "file-pages %" PRIu64 " < %u", dxbfile_pages, NUM_METAS);
    if (unlikely(usr->result.backed_pages < NUM_METAS))
      chk_scope_issue(inner, "backed-pages %zu < %u", usr->result.backed_pages, NUM_METAS);
    if (unlikely(usr->result.backed_pages < NUM_METAS)) {
      chk_scope_issue(inner, "backed-pages %zu < num-metas %u", usr->result.backed_pages, NUM_METAS);
      return MDBX_CORRUPTED;
    }
    if (unlikely(dxbfile_pages < NUM_METAS)) {
      chk_scope_issue(inner, "backed-pages %zu < num-metas %u", usr->result.backed_pages, NUM_METAS);
      return MDBX_CORRUPTED;
    }
    if (unlikely(usr->result.backed_pages > (size_t)MAX_PAGENO + 1)) {
      chk_scope_issue(inner, "backed-pages %zu > max-pages %zu", usr->result.backed_pages, (size_t)MAX_PAGENO + 1);
      usr->result.backed_pages = MAX_PAGENO + 1;
    }

    if ((env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) {
      if (unlikely(usr->result.backed_pages > dxbfile_pages)) {
        chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, usr->result.backed_pages, dxbfile_pages);
        usr->result.backed_pages = (size_t)dxbfile_pages;
      }
      if (unlikely(usr->result.alloc_pages > usr->result.backed_pages)) {
        chk_scope_issue(scope, "alloc-pages %zu > backed-pages %zu", usr->result.alloc_pages, usr->result.backed_pages);
        usr->result.alloc_pages = usr->result.backed_pages;
      }
    } else {
      /* DB may be shrunk by writer down to the allocated (but unused) pages. */
      if (unlikely(usr->result.alloc_pages > usr->result.backed_pages)) {
        chk_scope_issue(inner, "alloc-pages %zu > backed-pages %zu", usr->result.alloc_pages, usr->result.backed_pages);
        usr->result.alloc_pages = usr->result.backed_pages;
      }
      if (unlikely(usr->result.alloc_pages > dxbfile_pages)) {
        chk_scope_issue(inner, "alloc-pages %zu > file-pages %" PRIu64, usr->result.alloc_pages, dxbfile_pages);
        usr->result.alloc_pages = (size_t)dxbfile_pages;
      }
      if (unlikely(usr->result.backed_pages > dxbfile_pages))
        usr->result.backed_pages = (size_t)dxbfile_pages;
    }

    line = chk_line_feed(chk_print(chk_line_begin(inner, MDBX_chk_info), "pagesize %u (%u system), max keysize %u..%u",
                                   env->ps, globals.sys_pagesize, mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT),
                                   mdbx_env_get_maxkeysize_ex(env, MDBX_DB_DEFAULTS)));
    if ((env->flags & MDBX_EXCLUSIVE) == 0 && env->lck_mmap.lck) {
      line = chk_line_feed(chk_print(chk_line_begin(inner, MDBX_chk_info), "currently %u readers of %u maximum",
                                     atomic_load32(&env->lck_mmap.lck->rdt_length, mo_Relaxed), env->max_readers));
    }
    line = chk_line_feed(chk_print_size(line, "mapsize ", env->dxb_mmap.current, nullptr));
    if (txn->geo.lower == txn->geo.upper)
      line = chk_print_size(line, "fixed datafile: ", chk->envinfo.mi_geo.current, nullptr);
    else {
      line = chk_print_size(line, "dynamic datafile: ", chk->envinfo.mi_geo.lower, nullptr);
      line = chk_print_size(line, " .. ", chk->envinfo.mi_geo.upper, ", ");
      line = chk_print_size(line, "+", chk->envinfo.mi_geo.grow, ", ");

      line = chk_line_feed(chk_print_size(line, "-", chk->envinfo.mi_geo.shrink, nullptr));
      line = chk_print_size(line, "current datafile: ", chk->envinfo.mi_geo.current, nullptr);
    }
    cASSERT0(txn, txn->geo.now == chk->envinfo.mi_geo.current / chk->envinfo.mi_dxb_pagesize);
    chk_line_end(chk_print(line, ", %u pages", txn->geo.now));
#if defined(_WIN32) || defined(_WIN64) || MDBX_DEBUG > 0
    if (txn->geo.shrink_pv && txn->geo.now != txn->geo.upper && scope->verbosity >= MDBX_chk_verbose) {
      line = chk_line_begin(inner, MDBX_chk_notice);
      chk_line_feed(chk_print(line, " > WARNING: Due Windows system limitations a file couldn't"));
      chk_line_feed(chk_print(line, " > be truncated while the database is opened. So, the size"));
      chk_line_feed(chk_print(line, " > database file of may by large than the database itself,"));
      chk_line_end(chk_print(line, " > until it will be closed or reopened in read-write mode."));
    }
#endif /* Windows || Debug */
    line = chk_print_size(chk_line_begin(scope, MDBX_chk_verbose), "filesystem: io-block ", chk->envinfo.mi_sys_ioblk,
                          ", space allocated for the dxb-file ");
    if (chk->envinfo.mi_dxb_fallocated == chk->envinfo.mi_geo.current) {
      line = chk_puts(line, "exactly");
    } else {
      line = chk_print_size(
          line, (chk->envinfo.mi_dxb_fallocated > chk->envinfo.mi_geo.current) ? "with excess " : "partially ",
          chk->envinfo.mi_dxb_fallocated, " ");
      ratio2digits_buffer_t buffer;
      line =
          chk_print(line, "%s%%", ratio2percent(chk->envinfo.mi_dxb_fallocated, chk->envinfo.mi_geo.current, &buffer));
    }
    chk_line_end(line);
    chk_verbose_meta(inner, 0);
    chk_verbose_meta(inner, 1);
    chk_verbose_meta(inner, 2);

    if (env->stuck_meta >= 0) {
      chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_processing),
                             "skip checking meta-pages since the %u"
                             " is selected for verification",
                             env->stuck_meta));
      line = chk_line_feed(chk_print(chk_line_begin(inner, MDBX_chk_resolution),
                                     "transactions: recent %" PRIu64 ", "
                                     "selected for verification %" PRIu64 ", lag %" PRIi64,
                                     chk->envinfo.mi_recent_txnid, chk->envinfo.mi_meta_txnid[env->stuck_meta],
                                     chk->envinfo.mi_recent_txnid - chk->envinfo.mi_meta_txnid[env->stuck_meta]));
      chk_line_end(line);
    } else {
      chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), "performs check for meta-pages clashes"));
      const unsigned meta_clash_mask = meta_eq_mask(&chk->troika);
      if (meta_clash_mask & 1)
        chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 0, 1);
      if (meta_clash_mask & 2)
        chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 1, 2);
      if (meta_clash_mask & 4)
        chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 2, 0);

      const unsigned prefer_steady_metanum = chk->troika.prefer_steady;
      const uint64_t prefer_steady_txnid = chk->troika.txnid[prefer_steady_metanum];
      const unsigned recent_metanum = chk->troika.recent;
      const uint64_t recent_txnid = chk->troika.txnid[recent_metanum];
      if (env->flags & MDBX_EXCLUSIVE) {
        chk_line_end(
            chk_puts(chk_line_begin(inner, MDBX_chk_verbose), "performs full check recent-txn-id with meta-pages"));
        eASSERT0(env, recent_txnid == chk->envinfo.mi_recent_txnid);
        if (prefer_steady_txnid != recent_txnid) {
          if ((chk->flags & MDBX_CHK_READWRITE) != 0 && (env->flags & MDBX_RDONLY) == 0 &&
              recent_txnid > prefer_steady_txnid &&
              (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) != 0 &&
              chk->envinfo.mi_bootid.current.x == chk->envinfo.mi_bootid.meta[recent_metanum].x &&
              chk->envinfo.mi_bootid.current.y == chk->envinfo.mi_bootid.meta[recent_metanum].y) {
            chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_verbose),
                                   "recent meta-%u is weak, but boot-id match current"
                                   " (will synced upon successful check)",
                                   recent_metanum));
          } else
            chk_scope_issue(inner, "steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 ")",
                            prefer_steady_metanum, prefer_steady_txnid, recent_txnid);
        }
      } else if (chk->write_locked) {
        chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose),
                              "performs lite check recent-txn-id with meta-pages (not a "
                              "monopolistic mode)"));
        if (recent_txnid != chk->envinfo.mi_recent_txnid) {
          chk_scope_issue(inner, "weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 ")",
                          recent_metanum, recent_txnid, chk->envinfo.mi_recent_txnid);
        }
      } else {
        chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose),
                              "skip check recent-txn-id with meta-pages (monopolistic or "
                              "read-write mode only)"));
      }

      chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_resolution),
                             "transactions: recent %" PRIu64 ", latter reader %" PRIu64 ", lag %" PRIi64,
                             chk->envinfo.mi_recent_txnid, chk->envinfo.mi_latter_reader_txnid,
                             chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid));
    }
  }
  err = chk_scope_restore(scope, err);

  //--------------------------------------------------------------------------

  const char *const subj_tree = "B-Trees";
  if (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL)
    chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skipping %s traversal...", subj_tree));
  else {
    err = chk_scope_begin(chk, -1, MDBX_chk_tree, nullptr, &usr->result.tree_problems,
                          "Traversal %s by txn#%" PRIaTXN "...", subj_tree, txn->txnid);
    if (likely(!err))
      err = chk_tree(usr->scope);
    if (usr->result.tree_problems && usr->result.gc_tree_problems == 0)
      usr->result.gc_tree_problems = usr->result.tree_problems;
    if (usr->result.tree_problems && usr->result.kv_tree_problems == 0)
      usr->result.kv_tree_problems = usr->result.tree_problems;
    if (usr->result.histogram_page_age.count) {
      line = chk_line_begin(scope, MDBX_chk_info);
      histogram_dist(line, &usr->result.histogram_page_age, "pages age distribution", "dirty", false);
      chk_line_end(line);
    }
    if (usr->result.histogram_pgno_payload.count) {
      line = chk_line_begin(scope, MDBX_chk_info);
      histogram_dist(line, &usr->result.histogram_pgno_payload, "payload pgno distribution", "bad", false);
      chk_line_end(line);
    }
    if (usr->result.histogram_pgno_retained.count) {
      line = chk_line_begin(scope, MDBX_chk_info);
      histogram_dist(line, &usr->result.histogram_pgno_retained, "retained pgno distribution", "bad", false);
      chk_line_end(line);
    }
    chk_scope_restore(scope, err);
  }

  const char *const subj_gc = chk_v2a(chk, MDBX_CHK_GC);
  if (usr->result.gc_tree_problems > 0)
    chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing),
                           "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", subj_gc, subj_tree,
                           usr->result.problems_gc = usr->result.gc_tree_problems));
  else {
    err = chk_scope_begin(chk, -1, MDBX_chk_gc, &chk->table_gc, &usr->result.problems_gc,
                          "Processing %s by txn#%" PRIaTXN "...", subj_gc, txn->txnid);
    if (likely(!err))
      err = chk_db(usr->scope, FREE_DBI, &chk->table_gc, chk_handle_gc);
    line = chk_line_begin(scope, MDBX_chk_info);
    histogram_print(scope, line, &chk->table_gc.histogram.nested_height_or_gc_span_length, "span(s)", "single", false);
    chk_line_end(line);
    if (usr->result.problems_gc == 0 && (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) == 0) {
      const size_t used_pages = usr->result.alloc_pages - usr->result.gc_pages;
      if (usr->result.processed_pages != used_pages)
        chk_scope_issue(usr->scope, "used pages mismatch (%" PRIuSIZE "(walked) != %" PRIuSIZE "(allocated - GC))",
                        usr->result.processed_pages, used_pages);
      if (usr->result.unused_pages != usr->result.gc_pages)
        chk_scope_issue(usr->scope, "GC pages mismatch (%" PRIuSIZE "(expected) != %" PRIuSIZE "(GC))",
                        usr->result.unused_pages, usr->result.gc_pages);
    }
  }
  chk_scope_restore(scope, err);

  //--------------------------------------------------------------------------

  err = chk_scope_begin(chk, 1, MDBX_chk_space, nullptr, nullptr, "Page allocation:");
  const size_t backed = usr->result.backed_pages;
  const size_t boundary = txn->geo.upper;
  const size_t detained = usr->result.gc_pages - usr->result.reclaimable_pages;
  const size_t available2boundary = boundary - usr->result.alloc_pages + usr->result.reclaimable_pages;
  const size_t available2backed = backed - usr->result.alloc_pages + usr->result.reclaimable_pages;
  const size_t remained2boundary = boundary - usr->result.alloc_pages;
  const size_t remained2backed = backed - usr->result.alloc_pages;

  const size_t used = (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) ? usr->result.alloc_pages - usr->result.gc_pages
                                                                   : usr->result.processed_pages;

  line = chk_line_begin(usr->scope, MDBX_chk_info);
  line = chk_print_pages_percent(line, "backed by file:\0 of boundary\0", backed, boundary);
  line = chk_print_pages_percent(line, ",\0\0 left to boundary", boundary - backed, boundary);
  line = chk_line_feed(line);

  line = chk_print_pages_percent_bb(line, "used:", used, backed, boundary);
  line = chk_line_feed(line);

  line = chk_print_pages_percent(line, "remained:\0\0 of backed", remained2backed, backed);
  line = chk_print_pages_percent(line, ", left\0\0 to boundary", remained2boundary, boundary);
  line = chk_line_feed(line);

  line = chk_print_pages_percent_bb(line, "reclaimable:", usr->result.reclaimable_pages, backed, boundary);
  line = chk_print_pages_percent_bb(line, ", within GC", usr->result.gc_pages, backed, boundary);
  line = chk_line_feed(line);

  line = chk_print_pages_percent_bb(line, "detained by reader(s):", detained, backed, boundary);
  line = chk_print(line, ", %u reader(s), lag %" PRIi64, chk->envinfo.mi_numreaders,
                   chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid);
  line = chk_line_feed(line);

  line = chk_print_pages_percent_bb(line, "allocated:", usr->result.alloc_pages, backed, boundary);
  line = chk_line_feed(line);

  line = chk_print_pages_percent(line, "available:\0 of backed\0", available2backed, backed);
  line = chk_print_pages_percent(line, ", left\0\0 to boundary", available2boundary, boundary);
  chk_line_end(line);

  line = chk_line_begin(usr->scope, MDBX_chk_resolution);
  line = chk_print(line, "%s %zu pages", (boundary == txn->geo.now) ? "total" : "upto", boundary);
  line = chk_print_pages_percent(line, ", backed\0\0", backed, boundary);
  line = chk_print_pages_percent(line, ", allocated\0\0", usr->result.alloc_pages, boundary);
  line = chk_print_pages_percent(line, ", available\0\0", available2boundary, boundary);
  chk_line_end(line);
  chk_scope_restore(scope, err);

  //--------------------------------------------------------------------------

  const char *const subj_main = chk_v2a(chk, MDBX_CHK_MAIN);
  if (chk->flags & MDBX_CHK_SKIP_KV_TRAVERSAL)
    chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s...", subj_main));
  else if ((usr->result.problems_kv = usr->result.kv_tree_problems) > 0)
    chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing),
                           "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", subj_main, subj_tree,
                           usr->result.problems_kv = usr->result.kv_tree_problems));
  else {
    err = chk_scope_begin(chk, 0, MDBX_chk_maindb, &chk->table_main, &usr->result.problems_kv, "Processing %s...",
                          subj_main);
    if (likely(!err))
      err = chk_db(usr->scope, MAIN_DBI, &chk->table_main, chk_handle_kv);
    chk_scope_restore(scope, err);

    const char *const subj_tables = "table(s)";
    if (usr->result.problems_kv && usr->result.table_total)
      chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s", subj_tables));
    else if (usr->result.problems_kv == 0 && usr->result.table_total == 0)
      chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "No %s", subj_tables));
    else if (usr->result.problems_kv == 0 && usr->result.table_total) {
      err = chk_scope_begin(chk, 1, MDBX_chk_tables, nullptr, &usr->result.problems_kv,
                            "Processing %s by txn#%" PRIaTXN "...", subj_tables, txn->txnid);
      if (!err)
        err = chk_db(usr->scope, MAIN_DBI, &chk->table_main, nullptr);
      if (usr->scope->subtotal_issues)
        chk_line_end(chk_print(chk_line_begin(usr->scope, MDBX_chk_resolution),
                               "processed %" PRIuSIZE " of %" PRIuSIZE " %s, %" PRIuSIZE " problems(s)",
                               usr->result.table_processed, usr->result.table_total, subj_tables,
                               usr->scope->subtotal_issues));
    }
    chk_scope_restore(scope, err);
  }

  return chk_scope_end(chk, chk_scope_begin(chk, 0, MDBX_chk_conclude, nullptr, nullptr, nullptr));
}

__cold int mdbx_env_chk_encount_problem(MDBX_chk_context_t *ctx) {
  if (likely(ctx && ctx->internal && ctx->internal->usr == ctx && ctx->internal->problem_counter && ctx->scope)) {
    *ctx->internal->problem_counter += 1;
    ctx->scope->subtotal_issues += 1;
    return MDBX_SUCCESS;
  }
  return MDBX_EINVAL;
}

__cold int mdbx_env_chk(MDBX_env *env, const struct MDBX_chk_callbacks *cb, MDBX_chk_context_t *ctx,
                        const MDBX_chk_flags_t flags, MDBX_chk_severity_t verbosity, unsigned timeout_seconds_16dot16) {
  int err, rc = check_env(env, false);
  if (unlikely(rc != MDBX_SUCCESS))
    return LOG_IFERR(rc);
  if (unlikely(!cb || !ctx || ctx->internal))
    return LOG_IFERR(MDBX_EINVAL);

  MDBX_chk_internal_t *const chk = osal_calloc(1, sizeof(MDBX_chk_internal_t));
  if (unlikely(!chk))
    return LOG_IFERR(MDBX_ENOMEM);

  chk->cb = cb;
  chk->usr = ctx;
  chk->usr->internal = chk;
  chk->usr->env = env;
  chk->flags = flags;

  chk->table_gc.id = -1;
  chk->table_gc.name.iov_base = MDBX_CHK_GC;
  chk->table[FREE_DBI] = &chk->table_gc;

  chk->table_main.id = -1;
  chk->table_main.name.iov_base = MDBX_CHK_MAIN;
  chk->table[MAIN_DBI] = &chk->table_main;

  chk->monotime_timeout =
      timeout_seconds_16dot16 ? osal_16dot16_to_monotime(timeout_seconds_16dot16) + osal_monotime() : 0;
  chk->usr->scope_nesting = 0;
  chk->usr->result.tables = (const void *)&chk->table;

  MDBX_chk_scope_t *const top = chk->scope_stack;
  top->verbosity = verbosity;
  top->internal = chk;

  // init
  rc = chk_scope_end(chk, chk_scope_begin(chk, 0, MDBX_chk_init, nullptr, nullptr, nullptr));

  // lock
  if (likely(!rc))
    rc = chk_scope_begin(chk, 0, MDBX_chk_lock, nullptr, nullptr, "Taking %slock...",
                         (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) ? "" : "read ");
  if (likely(!rc) && (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0 && (flags & MDBX_CHK_READWRITE)) {
    rc = mdbx_txn_lock(env, false);
    if (unlikely(rc))
      chk_error_rc(ctx->scope, rc, "mdbx_txn_lock");
    else
      chk->write_locked = true;
  }
  if (likely(!rc)) {
    rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &ctx->txn);
    if (unlikely(rc))
      chk_error_rc(ctx->scope, rc, "mdbx_txn_begin");
  }
  chk_scope_end(chk, rc);

  // doit
  if (likely(!rc)) {
    chk->table_gc.flags = ctx->txn->dbs[FREE_DBI].flags;
    chk->table_main.flags = ctx->txn->dbs[MAIN_DBI].flags;
    rc = env_chk(top);
  }

  // unlock
  if (ctx->txn || chk->write_locked) {
    chk_scope_begin(chk, 0, MDBX_chk_unlock, nullptr, nullptr, nullptr);
    if (ctx->txn) {
      err = mdbx_txn_abort(ctx->txn);
      if (err && !rc)
        rc = err;
      ctx->txn = nullptr;
    }
    if (chk->write_locked)
      mdbx_txn_unlock(env);
    rc = chk_scope_end(chk, rc);
  }

  // finalize
  err = chk_scope_begin(chk, 0, MDBX_chk_finalize, nullptr, nullptr, nullptr);
  rc = chk_scope_end(chk, err ? err : rc);
  chk_dispose(chk);
  return LOG_IFERR(rc);
}

/*------------------------------------------------------------------------------
 * Pack/Unpack 16-bit values for Grow step & Shrink threshold */

MDBX_NOTHROW_CONST_FUNCTION static inline pgno_t me2v(size_t m, size_t e) {
  ASSERT(m < 2048 && e < 8);
  return (pgno_t)(32768 + ((m + 1) << (e + 8)));
}

MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t v2me(size_t v, size_t e) {
  ASSERT(v > (e ? me2v(2047, e - 1) : 32768));
  ASSERT(v <= me2v(2047, e));
  size_t m = (v - 32768 + ((size_t)1 << (e + 8)) - 1) >> (e + 8);
  m -= m > 0;
  ASSERT(m < 2048 && e < 8);
  // f e d c b a 9 8 7 6 5 4 3 2 1 0
  // 1 e e e m m m m m m m m m m m 1
  const uint16_t pv = (uint16_t)(0x8001 + (e << 12) + (m << 1));
  ASSERT(pv != 65535);
  return pv;
}

/* Convert 16-bit packed (exponential quantized) value to number of pages */
pgno_t pv2pages(uint16_t pv) {
  if ((pv & 0x8001) != 0x8001)
    return pv;
  if (pv == 65535)
    return 65536;
  // f e d c b a 9 8 7 6 5 4 3 2 1 0
  // 1 e e e m m m m m m m m m m m 1
  return me2v((pv >> 1) & 2047, (pv >> 12) & 7);
}

/* Convert number of pages to 16-bit packed (exponential quantized) value */
uint16_t pages2pv(size_t pages) {
  if (pages < 32769 || (pages < 65536 && (pages & 1) == 0))
    return (uint16_t)pages;
  if (pages <= me2v(2047, 0))
    return v2me(pages, 0);
  if (pages <= me2v(2047, 1))
    return v2me(pages, 1);
  if (pages <= me2v(2047, 2))
    return v2me(pages, 2);
  if (pages <= me2v(2047, 3))
    return v2me(pages, 3);
  if (pages <= me2v(2047, 4))
    return v2me(pages, 4);
  if (pages <= me2v(2047, 5))
    return v2me(pages, 5);
  if (pages <= me2v(2047, 6))
    return v2me(pages, 6);
  return (pages < me2v(2046, 7)) ? v2me(pages, 7) : 65533;
}

__cold bool pv2pages_verify(void) {
  bool ok = true, dump_translation = false;
  for (size_t i = 0; i < 65536; ++i) {
    size_t pages = pv2pages(i);
    size_t x = pages2pv(pages);
    size_t xp = pv2pages(x);
    if (pages != xp) {
      ERROR("%zu => %zu => %zu => %zu\n", i, pages, x, xp);
      ok = false;
    } else if (dump_translation && !(x == i || (x % 2 == 0 && x < 65536))) {
      DEBUG("%zu => %zu => %zu => %zu\n", i, pages, x, xp);
    }
  }
  return ok;
}

/*----------------------------------------------------------------------------*/

MDBX_NOTHROW_PURE_FUNCTION size_t bytes_ceil2sp_bytes(const MDBX_env *env, size_t bytes) {
  return ceil_powerof2(bytes, (env->ps > globals.sys_pagesize) ? env->ps : globals.sys_pagesize);
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION size_t bytes_ceil2sp_pgno(const MDBX_env *env, size_t bytes) {
  return bytes2pgno(env, bytes_ceil2sp_bytes(env, bytes));
}

MDBX_NOTHROW_PURE_FUNCTION size_t pgno_ceil2sp_bytes(const MDBX_env *env, size_t pgno) {
  return ceil_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize);
}

MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_ceil2sp_pgno(const MDBX_env *env, size_t pgno) {
  return bytes2pgno(env, pgno_ceil2sp_bytes(env, pgno));
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION size_t bytes_ceil2os_bytes(const MDBX_env *env, size_t bytes) {
  const size_t sys_unit =
      MDBX_ROUNDING_TO_ALLOCATION_GRANULARITY ? globals.sys_allocation_granularity : globals.sys_pagesize;
  return ceil_powerof2(bytes, (env->ps > sys_unit) ? env->ps : sys_unit);
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION size_t bytes_ceil2os_pgno(const MDBX_env *env, size_t bytes) {
  return bytes2pgno(env, bytes_ceil2os_bytes(env, bytes));
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION size_t pgno_ceil2os_bytes(const MDBX_env *env, size_t pgno) {
  const size_t sys_unit =
      MDBX_ROUNDING_TO_ALLOCATION_GRANULARITY ? globals.sys_allocation_granularity : globals.sys_pagesize;
  return ceil_powerof2(pgno2bytes(env, pgno), sys_unit);
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_ceil2os_pgno(const MDBX_env *env, size_t pgno) {
  return bytes2pgno(env, pgno_ceil2os_bytes(env, pgno));
}

/*----------------------------------------------------------------------------*/

MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint_inline(const size_t expected_alignment,
                                                                      const MDBX_val *a, const MDBX_val *b) {
  if (likely(a->iov_len == b->iov_len)) {
    if (sizeof(size_t) > 7 && likely(a->iov_len == 8))
      return cmp_uint64_unchecked(expected_alignment, a, b);
    if (likely(a->iov_len == 4))
      return cmp_uint32_unchecked(expected_alignment, a, b);
    if (sizeof(size_t) < 8 && likely(a->iov_len == 8))
      return cmp_uint64_unchecked(expected_alignment, a, b);
  }
  ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", a->iov_base, a->iov_len, b->iov_base,
        b->iov_len);
  return 0;
}

MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint32_inline(const size_t expected_alignment,
                                                                        const MDBX_val *a, const MDBX_val *b) {
  if (likely(a->iov_len == b->iov_len && a->iov_len == 4))
    return cmp_uint32_unchecked(expected_alignment, a, b);
  ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", a->iov_base, a->iov_len, b->iov_base,
        b->iov_len);
  return 0;
}

MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_uint64_inline(const size_t expected_alignment,
                                                                        const MDBX_val *a, const MDBX_val *b) {
  if (likely(a->iov_len == b->iov_len && a->iov_len == 8))
    return cmp_uint64_unchecked(expected_alignment, a, b);
  ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", a->iov_base, a->iov_len, b->iov_base,
        b->iov_len);
  return 0;
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint_unaligned(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint_inline(1, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint32_unaligned(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint32_inline(1, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint64_unaligned(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint64_inline(1, a, b);
}

#ifndef cmp_uint_align2
/* Compare two items pointing at 2-byte aligned unsigned int's. */
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint_align2(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint_inline(2, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint32_align2(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint32_inline(2, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint64_align2(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint64_inline(2, a, b);
}
#endif /* cmp_uint_align2 */

#ifndef cmp_uint_align4
/* Compare two items pointing at 4-byte aligned unsigned int's. */
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint_align4(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint_inline(4, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint32_align4(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint32_inline(4, a, b);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_uint64_align4(const MDBX_val *a, const MDBX_val *b) {
  return cmp_uint64_inline(4, a, b);
}
#endif /* cmp_uint_align4 */

MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_len(size_t a, size_t b) {
  const intptr_t diff_len = a - b;
  ASSERT(diff_len == (int)diff_len);
  /* кастинг допустим, так как длина ключей проверяется и не должна превышать MAX_INT. */
  return (int)diff_len;
}

/* Compare two items lexically */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a, const MDBX_val *b) {
  const int diff_len = cmp_len(a->iov_len, b->iov_len);
  const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
  int diff_data = shortest ? memcmp(a->iov_base, b->iov_base, shortest) : 0;
  return likely(diff_data) ? diff_data : diff_len;
}

MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned tail3le(const uint8_t *p, size_t l) {
  STATIC_ASSERT(sizeof(unsigned) > 2);
  // 1: 0 0 0
  // 2: 0 1 1
  // 3: 0 1 2
  return p[0] | p[l >> 1] << 8 | p[l - 1] << 16;
}

/* Compare two items in reverse byte order */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a, const MDBX_val *b) {
  size_t left = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
  if (likely(left)) {
    const uint8_t *pa = ptr_disp(a->iov_base, a->iov_len);
    const uint8_t *pb = ptr_disp(b->iov_base, b->iov_len);
    while (left >= sizeof(size_t)) {
      pa -= sizeof(size_t);
      pb -= sizeof(size_t);
      left -= sizeof(size_t);
      STATIC_ASSERT(sizeof(size_t) == 4 || sizeof(size_t) == 8);
      if (sizeof(size_t) == 4) {
        uint32_t xa = unaligned_peek_u32(1, pa);
        uint32_t xb = unaligned_peek_u32(1, pb);
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
        xa = osal_bswap32(xa);
        xb = osal_bswap32(xb);
#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
        if (xa != xb)
          return (xa < xb) ? -1 : 1;
      } else {
        uint64_t xa = unaligned_peek_u64(1, pa);
        uint64_t xb = unaligned_peek_u64(1, pb);
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
        xa = osal_bswap64(xa);
        xb = osal_bswap64(xb);
#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
        if (xa != xb)
          return (xa < xb) ? -1 : 1;
      }
    }
    if (sizeof(size_t) == 8 && left >= 4) {
      pa -= 4;
      pb -= 4;
      left -= 4;
      uint32_t xa = unaligned_peek_u32(1, pa);
      uint32_t xb = unaligned_peek_u32(1, pb);
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
      xa = osal_bswap32(xa);
      xb = osal_bswap32(xb);
#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
      if (xa != xb)
        return (xa < xb) ? -1 : 1;
    }
    if (left) {
      unsigned xa = tail3le(pa - left, left);
      unsigned xb = tail3le(pb - left, left);
      if (xa != xb)
        return (xa < xb) ? -1 : 1;
    }
  }
  return cmp_len(a->iov_len, b->iov_len);
}

/* Fast non-lexically comparator */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lenfast(const MDBX_val *a, const MDBX_val *b) {
  int diff = cmp_len(a->iov_len, b->iov_len);
  return (likely(diff) || a->iov_len == 0) ? diff : memcmp(a->iov_base, b->iov_base, a->iov_len);
}

MDBX_NOTHROW_PURE_FUNCTION __hot bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) {
  if (likely(l > 3)) {
    if (MDBX_UNALIGNED_OK >= 4 && likely(l < 9))
      return ((unaligned_peek_u32(1, a) - unaligned_peek_u32(1, b)) |
              (unaligned_peek_u32(1, a + l - 4) - unaligned_peek_u32(1, b + l - 4))) == 0;
    if (MDBX_UNALIGNED_OK >= 8 && sizeof(size_t) > 7 && likely(l < 17))
      return ((unaligned_peek_u64(1, a) - unaligned_peek_u64(1, b)) |
              (unaligned_peek_u64(1, a + l - 8) - unaligned_peek_u64(1, b + l - 8))) == 0;
    return memcmp(a, b, l) == 0;
  }
  if (likely(l))
    return tail3le(a, l) == tail3le(b, l);
  return true;
}

int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : 1; }

int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : -1; }

/*----------------------------------------------------------------------------*/

__cold void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno, const bool lock_not_release) {
  for (;;) {
    const pgno_t mlock_pgno_before = atomic_load32(&env->mlocked_pgno, mo_AcquireRelease);
    eASSERT0(env, pgno_ceil2sp_pgno(env, mlock_pgno_before) == mlock_pgno_before);
    eASSERT0(env, pgno_ceil2sp_pgno(env, new_aligned_mlocked_pgno) == new_aligned_mlocked_pgno);
    if (lock_not_release ? (mlock_pgno_before >= new_aligned_mlocked_pgno)
                         : (mlock_pgno_before <= new_aligned_mlocked_pgno))
      break;
    if (likely(atomic_cas32(&((MDBX_env *)env)->mlocked_pgno, mlock_pgno_before, new_aligned_mlocked_pgno)))
      for (;;) {
        mdbx_atomic_uint32_t *const mlcnt = env->lck->mlcnt;
        const int32_t snap_locked = atomic_load32(mlcnt + 0, mo_Relaxed);
        const int32_t snap_unlocked = atomic_load32(mlcnt + 1, mo_Relaxed);
        if (mlock_pgno_before == 0 && (snap_locked - snap_unlocked) < INT_MAX) {
          eASSERT0(env, lock_not_release);
          if (unlikely(!atomic_cas32(mlcnt + 0, snap_locked, snap_locked + 1)))
            continue;
        }
        if (new_aligned_mlocked_pgno == 0 && (snap_locked - snap_unlocked) > 0) {
          eASSERT0(env, !lock_not_release);
          if (unlikely(!atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1)))
            continue;
        }
        NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u", lock_not_release ? "lock" : "unlock",
               lock_not_release ? mlock_pgno_before : new_aligned_mlocked_pgno,
               lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before, snap_locked - snap_unlocked,
               atomic_load32(mlcnt + 0, mo_Relaxed) - atomic_load32(mlcnt + 1, mo_Relaxed));
        return;
      }
  }
}

__cold void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes) {
  if (atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) > aligned_pgno) {
    int err = MDBX_ENOSYS;
    const size_t munlock_begin = pgno2bytes(env, aligned_pgno);
    const size_t munlock_size = end_bytes - munlock_begin;
    eASSERT0(env, end_bytes % globals.sys_pagesize == 0 && munlock_begin % globals.sys_pagesize == 0 &&
                      munlock_size % globals.sys_pagesize == 0);
#if defined(_WIN32) || defined(_WIN64)
    err = VirtualUnlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? MDBX_SUCCESS : (int)GetLastError();
    if (err == ERROR_NOT_LOCKED)
      err = MDBX_SUCCESS;
#elif defined(_POSIX_MEMLOCK_RANGE)
    err = munlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? errno : MDBX_SUCCESS;
#endif
    if (likely(err == MDBX_SUCCESS))
      update_mlcnt(env, aligned_pgno, false);
    else {
#if defined(_WIN32) || defined(_WIN64)
      WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size, err);
#else
      WARNING("munlock(%zu, %zu) error %d", munlock_begin, munlock_size, err);
#endif
    }
  }
}

__cold void munlock_all(const MDBX_env *env) { munlock_after(env, 0, bytes_ceil2sp_bytes(env, env->dxb_mmap.current)); }

/*----------------------------------------------------------------------------*/

uint32_t combine_durability_flags(const uint32_t a, const uint32_t b) {
  uint32_t r = a | b;

  /* avoid false MDBX_UTTERLY_NOSYNC */
  if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC))
    r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC;

  /* convert DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */
  if ((r & (MDBX_WRITEMAP | DEPRECATED_MAPASYNC)) == (MDBX_WRITEMAP | DEPRECATED_MAPASYNC) &&
      !F_ISSET(r, MDBX_UTTERLY_NOSYNC))
    r = (r - DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC;

  /* force MDBX_NOMETASYNC if NOSYNC enabled */
  if (r & (MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC))
    r |= MDBX_NOMETASYNC;

  ASSERT(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC)));
  return r;
}

/* check against https://libmdbx.dqdkfa.ru/dead-github/issues/269 */
static bool coherency_check(const MDBX_env *env, const txnid_t txnid, const volatile tree_t *trees,
                            const volatile meta_t *meta, bool report) {
  const txnid_t freedb_mod_txnid = trees[FREE_DBI].mod_txnid;
  const txnid_t maindb_mod_txnid = trees[MAIN_DBI].mod_txnid;
  const pgno_t last_pgno = meta->geometry.now;

  const pgno_t freedb_root_pgno = trees[FREE_DBI].root;
  const page_t *freedb_root =
      (env->dxb_mmap.base && freedb_root_pgno < last_pgno) ? pgno2page(env, freedb_root_pgno) : nullptr;

  const pgno_t maindb_root_pgno = trees[MAIN_DBI].root;
  const page_t *maindb_root =
      (env->dxb_mmap.base && maindb_root_pgno < last_pgno) ? pgno2page(env, maindb_root_pgno) : nullptr;
  const uint64_t magic_and_version = unaligned_peek_u64_volatile(4, &meta->magic_and_version);

  bool ok = true;
  if (freedb_root_pgno != P_INVALID && unlikely(freedb_root_pgno >= last_pgno)) {
    if (report)
      WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "free", freedb_root_pgno, txnid,
              (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)"
                                    : "(wagering meta)");
    ok = false;
  }
  if (maindb_root_pgno != P_INVALID && unlikely(maindb_root_pgno >= last_pgno)) {
    if (report)
      WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "main", maindb_root_pgno, txnid,
              (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)"
                                    : "(wagering meta)");
    ok = false;
  }
  if (unlikely(txnid < freedb_mod_txnid ||
               (!freedb_mod_txnid && freedb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) {
    if (report)
      WARNING(
          "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "free", freedb_mod_txnid, txnid,
          (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)");
    ok = false;
  }
  if (unlikely(txnid < maindb_mod_txnid ||
               (!maindb_mod_txnid && maindb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) {
    if (report)
      WARNING(
          "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "main", maindb_mod_txnid, txnid,
          (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)");
    ok = false;
  }

  /* Проверяем отметки внутри корневых страниц только если сами страницы в пределах текущего отображения. */
  if (likely(freedb_root && freedb_mod_txnid &&
             (size_t)ptr_dist(env->dxb_mmap.base, freedb_root) < env->dxb_mmap.limit)) {
    VALGRIND_MAKE_MEM_DEFINED(freedb_root, sizeof(freedb_root->txnid));
    MDBX_ASAN_UNPOISON_MEMORY_REGION(freedb_root, sizeof(freedb_root->txnid));
    const txnid_t root_txnid = freedb_root->txnid;
    if (unlikely(root_txnid != freedb_mod_txnid)) {
      if (report)
        WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s",
                freedb_root_pgno, root_txnid, "free", freedb_mod_txnid,
                (env->stuck_meta < 0) ? "(workaround for incoherent flaw of "
                                        "unified page/buffer cache)"
                                      : "(wagering meta)");
      ok = false;
    }
  }
  if (likely(maindb_root && maindb_mod_txnid &&
             (size_t)ptr_dist(env->dxb_mmap.base, maindb_root) < env->dxb_mmap.limit)) {
    VALGRIND_MAKE_MEM_DEFINED(maindb_root, sizeof(maindb_root->txnid));
    MDBX_ASAN_UNPOISON_MEMORY_REGION(maindb_root, sizeof(maindb_root->txnid));
    const txnid_t root_txnid = maindb_root->txnid;
    if (unlikely(root_txnid != maindb_mod_txnid)) {
      if (report)
        WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s",
                maindb_root_pgno, root_txnid, "main", maindb_mod_txnid,
                (env->stuck_meta < 0) ? "(workaround for incoherent flaw of "
                                        "unified page/buffer cache)"
                                      : "(wagering meta)");
      ok = false;
    }
  }
  if (unlikely(!ok) && report)
    env->lck->pgops.incoherence.weak =
        (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1;
  return ok;
}

__cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env) {
  if (likely(timestamp && *timestamp == 0))
    *timestamp = osal_monotime();
  else if (unlikely(!timestamp || osal_monotime() - *timestamp > osal_16dot16_to_monotime(65536 / 10))) {
    if (pgno >= 0 && pgno != env->stuck_meta)
      ERROR("bailout waiting for %" PRIuSIZE " page arrival %s", pgno,
            "(workaround for incoherent flaw of unified page/buffer cache)");
    else if (env->stuck_meta < 0)
      ERROR("bailout waiting for valid snapshot (%s)", "workaround for incoherent flaw of unified page/buffer cache");
    return MDBX_PROBLEM;
  }

  osal_memory_fence(mo_AcquireRelease, true);
  osal_yield();
  return MDBX_RESULT_TRUE;
}

/* check with timeout as the workaround
 * for https://libmdbx.dqdkfa.ru/dead-github/issues/269 */
__hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp) {
  /* Copy the DB info and flags */
  txn->txnid = head.txnid;
  txn->geo = head.ptr_c->geometry;
  memcpy(txn->dbs, &head.ptr_c->trees, sizeof(head.ptr_c->trees));
  STATIC_ASSERT(sizeof(head.ptr_c->trees) == CORE_DBS * sizeof(tree_t));
  VALGRIND_MAKE_MEM_UNDEFINED(txn->dbs + CORE_DBS, txn->env->max_dbi - CORE_DBS);
  txn->canary = head.ptr_c->canary;

  if (unlikely(!coherency_check(txn->env, head.txnid, txn->dbs, head.ptr_v, *timestamp == 0) ||
               txn->txnid != meta_txnid(head.ptr_v)))
    return coherency_timeout(timestamp, -1, txn->env);

  if (unlikely(txn->dbs[FREE_DBI].flags != MDBX_INTEGERKEY)) {
    if ((txn->dbs[FREE_DBI].flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY ||
        unaligned_peek_u64(4, &head.ptr_c->magic_and_version) == MDBX_DATA_MAGIC) {
      ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, "GC/FreeDB");
      return MDBX_INCOMPATIBLE;
    }
    txn->dbs[FREE_DBI].flags &= DB_PERSISTENT_FLAGS;
  }
  cASSERT0(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY);
  tASSERT1(txn, check_table_flags(txn->dbs[MAIN_DBI].flags));
  return MDBX_SUCCESS;
}

int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta, const intptr_t pgno,
                            uint64_t *timestamp) {
  const bool report = !(timestamp && *timestamp);
  const txnid_t head_txnid = meta_txnid(meta);
  if (likely(head_txnid >= MIN_TXNID && head_txnid >= txnid)) {
    if (likely(coherency_check(env, head_txnid, &meta->trees.gc, meta, report))) {
      eASSERT0(env, meta->trees.gc.flags == MDBX_INTEGERKEY);
      eASSERT1(env, check_table_flags(meta->trees.main.flags));
      return MDBX_SUCCESS;
    }
  } else if (report) {
    env->lck->pgops.incoherence.weak =
        (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1;
    WARNING("catch %s txnid %" PRIaTXN " for meta_%" PRIaPGNO " %s",
            (head_txnid < MIN_TXNID) ? "invalid" : "unexpected", head_txnid,
            bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)),
            "(workaround for incoherent flaw of unified page/buffer cache)");
  }
  return coherency_timeout(timestamp, pgno, env);
}

bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report) {
  uint64_t timestamp = 0;
  return coherency_check_written(env, 0, meta, -1, report ? &timestamp : nullptr) == MDBX_SUCCESS;
}

__cold int cursor_validate(const MDBX_cursor *mc) {
  if (!mc->txn->wr.dirtylist) {
    cASSERT0(mc, (mc->txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
  } else {
    cASSERT0(mc, (mc->txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
    cASSERT0(mc, mc->txn->wr.dirtyroom + mc->txn->wr.dirtylist->length ==
                     (mc->txn->parent ? mc->txn->parent->wr.dirtyroom : mc->txn->env->options.dp_limit));
  }

  cASSERT0(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height : mc->top + 1 == mc->tree->height);
  if (unlikely((mc->checking & z_updating) ? mc->top + 1 > mc->tree->height : mc->top + 1 != mc->tree->height))
    return MDBX_CURSOR_FULL;

  if (is_pointed(mc) && (mc->checking & z_updating) == 0) {
    const page_t *mp = mc->pg[mc->top];
    const size_t nkeys = page_numkeys(mp);
    if (!is_hollow(mc)) {
      cASSERT0(mc, mc->ki[mc->top] < nkeys);
      if (mc->ki[mc->top] >= nkeys)
        return MDBX_CURSOR_FULL;
    }
    if (inner_pointed(mc)) {
      cASSERT0(mc, is_filled(mc));
      if (!is_filled(mc))
        return MDBX_CURSOR_FULL;
    }
  }

  for (intptr_t n = 0; n <= mc->top; ++n) {
    page_t *mp = mc->pg[n];
    const size_t nkeys = page_numkeys(mp);
    const bool expect_branch = (n < mc->tree->height - 1) ? true : false;
    const bool expect_nested_leaf = (n + 1 == mc->tree->height - 1) ? true : false;
    const bool branch = is_branch(mp) ? true : false;
    cASSERT0(mc, branch == expect_branch);
    if (unlikely(branch != expect_branch))
      return MDBX_CURSOR_FULL;
    if ((mc->checking & z_updating) == 0) {
      cASSERT0(mc, nkeys > mc->ki[n] || (!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0));
      if (unlikely(nkeys <= mc->ki[n] && !(!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0)))
        return MDBX_CURSOR_FULL;
    } else {
      cASSERT0(mc, nkeys + 1 >= mc->ki[n]);
      if (unlikely(nkeys + 1 < mc->ki[n]))
        return MDBX_CURSOR_FULL;
    }

    int err = page_check(mc, mp);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    for (size_t i = 0; i < nkeys; ++i) {
      if (branch) {
        node_t *node = page_node(mp, i);
        cASSERT0(mc, node_flags(node) == 0);
        if (unlikely(node_flags(node) != 0))
          return MDBX_CURSOR_FULL;
        pgno_t pgno = node_pgno(node);
        page_t *np;
        err = page_get(mc, pgno, &np, mp->txnid);
        cASSERT0(mc, err == MDBX_SUCCESS);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
        const bool nested_leaf = is_leaf(np) ? true : false;
        cASSERT0(mc, nested_leaf == expect_nested_leaf);
        if (unlikely(nested_leaf != expect_nested_leaf))
          return MDBX_CURSOR_FULL;
        err = page_check(mc, np);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
      }
    }
  }
  return MDBX_SUCCESS;
}

__cold int cursor_validate_updating(MDBX_cursor *mc) {
  const uint8_t checking = mc->checking;
  mc->checking |= z_updating;
  const int rc = cursor_validate(mc);
  mc->checking = checking;
  return rc;
}

bool cursor_is_tracked(const MDBX_cursor *mc) {
  for (MDBX_cursor *scan = mc->txn->cursors[cursor_dbi(mc)]; scan; scan = scan->next)
    if (mc == ((mc->flags & z_inner) ? &scan->subcur->cursor : scan))
      return true;
  return false;
}

/*----------------------------------------------------------------------------*/

static int touch_dbi(MDBX_cursor *mc) {
  cASSERT0(mc, (mc->flags & z_inner) == 0);
  cASSERT0(mc, (*cursor_dbi_state(mc) & DBI_DIRTY) == 0);
  *cursor_dbi_state(mc) |= DBI_DIRTY;
  mc->txn->flags |= MDBX_TXN_DIRTY;

  if (!cursor_is_core(mc)) {
    /* Touch DB record of named DB */
    cursor_couple_t cx;
    int rc = dbi_check(mc->txn, MAIN_DBI);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    rc = cursor_init(&cx.outer, mc->txn, MAIN_DBI);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    mc->txn->dbi_state[MAIN_DBI] |= DBI_DIRTY;
    rc = tree_search(&cx.outer, &container_of(mc->clc, kvx_t, clc)->name, Z_MODIFY);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }
  return MDBX_SUCCESS;
}

__hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data) {
  cASSERT0(mc, (mc->txn->flags & txn_ro_both) == 0);
  cASSERT0(mc, is_pointed(mc) || mc->tree->height == 0);
  cASSERT1(mc, cursor_is_tracked(mc));

  cASSERT0(mc, F_ISSET(dbi_state(mc->txn, FREE_DBI), DBI_LINDO | DBI_VALID));
  cASSERT0(mc, F_ISSET(dbi_state(mc->txn, MAIN_DBI), DBI_LINDO | DBI_VALID));
  if ((mc->flags & z_inner) == 0) {
    MDBX_txn *const txn = mc->txn;
    txn_dpl_lru_turn(txn);

    if (unlikely((*cursor_dbi_state(mc) & DBI_DIRTY) == 0)) {
      int err = touch_dbi(mc);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }

    /* Estimate how much space this operation will take: */
    /* 1) Max b-tree height, reasonable enough with including dups' sub-tree */
    size_t need = CURSOR_STACK_SIZE + 3;
    /* 2) GC/FreeDB for any payload */
    if (!cursor_is_gc(mc)) {
      need += txn->dbs[FREE_DBI].height + (size_t)3;
      /* 3) Named DBs also dirty the main DB */
      if (!cursor_is_main(mc))
        need += txn->dbs[MAIN_DBI].height + (size_t)3;
    }
#if xMDBX_DEBUG_SPILLING != 2
    /* production mode */
    /* 4) Double the page chain estimation
     * for extensively splitting, rebalance and merging */
    need += need;
    /* 5) Factor the key+data which to be put in */
    need += bytes2pgno(txn->env, node_size(key, data)) + (size_t)1;
#else
    /* debug mode */
    (void)key;
    (void)data;
    txn->env->debug_dirtied_est = ++need;
    txn->env->debug_dirtied_act = 0;
#endif /* xMDBX_DEBUG_SPILLING == 2 */

    int err = txn_spill(txn, mc, need);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }

  if (likely(is_pointed(mc)) && ((mc->txn->flags & MDBX_TXN_SPILLS) || !is_modifable(mc->txn, mc->pg[mc->top]))) {
    const int8_t top = mc->top;
    mc->top = 0;
    do {
      int err = page_touch(mc);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      mc->top += 1;
    } while (mc->top <= top);
    mc->top = top;
  }
  return MDBX_SUCCESS;
}

/*----------------------------------------------------------------------------*/

int cursor_shadow(MDBX_cursor *cursor, MDBX_txn *nested, const size_t dbi) {
  tASSERT0(nested, cursor->signature == cur_signature_live);
  tASSERT0(nested, cursor->txn != nested);
  cASSERT0(cursor, cursor->txn->flags & txn_may_have_cursors);
  cASSERT0(cursor, dbi == cursor_dbi(cursor));
  tASSERT0(nested, dbi > FREE_DBI && dbi < nested->n_dbi);

  const size_t size = cursor->subcur ? sizeof(MDBX_cursor) + sizeof(subcur_t) : sizeof(MDBX_cursor);
  MDBX_cursor *const shadow = osal_malloc(size);
  if (unlikely(!shadow))
    return MDBX_ENOMEM;

#if MDBX_DEBUG > 0
  memset(shadow, 0xCD, size);
  VALGRIND_MAKE_MEM_UNDEFINED(shadow, size);
#endif /* MDBX_DEBUG */
  *shadow = *cursor;
  cursor->backup = shadow;
  cursor->txn = nested;
  cursor->tree = &nested->dbs[dbi];
  cursor->dbi_state = &nested->dbi_state[dbi];
  subcur_t *subcur = cursor->subcur;
  if (subcur) {
    *(subcur_t *)(shadow + 1) = *subcur;
    subcur->cursor.txn = nested;
    subcur->cursor.dbi_state = &nested->dbi_state[dbi];
  }
  return MDBX_SUCCESS;
}

MDBX_cursor *cursor_eot(MDBX_cursor *cursor, MDBX_txn *txn) {
  MDBX_cursor *const next = cursor->next;
  const unsigned stage = cursor->signature;
  MDBX_cursor *const shadow = cursor->backup;
  ENSURE_OBJ(txn, stage == cur_signature_live || stage == cur_signature_wait4eot);
  cASSERT0(txn, cursor->txn == txn);
  if (shadow) {
    subcur_t *subcur = cursor->subcur;
    cASSERT0(txn, txn->parent != nullptr && shadow->txn == txn->parent);
    /* Zap: Using uninitialized memory '*subcur->backup'. */
    MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001);
    ENSURE_OBJ(txn, shadow->signature == cur_signature_live);
    cASSERT0(txn, subcur == shadow->subcur);
    if ((txn->flags & MDBX_TXN_ERROR) == 0) {
      /* Update pointers to parent txn */
      cursor->next = shadow->next;
      cursor->backup = shadow->backup;
      cursor->txn = shadow->txn;
      cursor->tree = shadow->tree;
      cursor->dbi_state = shadow->dbi_state;
      if (subcur) {
        subcur->cursor.txn = shadow->txn;
        subcur->cursor.dbi_state = shadow->dbi_state;
      }
    } else {
      /* Restore from backup, i.e. rollback/abort nested txn */
      *cursor = *shadow;
      cursor->signature = stage /* Promote (cur_signature_wait4eot) state to parent txn */;
      if (subcur)
        *subcur = *(subcur_t *)(shadow + 1);
    }
    shadow->signature = 0;
    osal_free(shadow);
  } else {
    cursor_drown((cursor_couple_t *)cursor);
    cursor->next = cursor;
    if (stage == cur_signature_wait4eot) {
      cursor->signature = 0;
      osal_free(cursor);
    } else
      cursor->signature = cur_signature_ready4dispose /* Cursor may be reused */;
  }
  return next;
}

/*----------------------------------------------------------------------------*/

static __always_inline int couple_init(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree,
                                       kvx_t *const kvx, uint8_t *const dbi_state) {

  VALGRIND_MAKE_MEM_UNDEFINED(couple, sizeof(cursor_couple_t));
  cASSERT0(txn, F_ISSET(*dbi_state, DBI_VALID | DBI_LINDO));

  couple->outer.signature = cur_signature_live;
  couple->outer.next = &couple->outer;
  couple->outer.backup = nullptr;
  couple->outer.txn = (MDBX_txn *)txn;
  couple->outer.tree = tree;
  couple->outer.clc = &kvx->clc;
  couple->outer.dbi_state = dbi_state;
  couple->outer.top_and_flags = z_fresh_mark;
  STATIC_ASSERT((int)z_branch == P_BRANCH && (int)z_leaf == P_LEAF && (int)z_largepage == P_LARGE &&
                (int)z_dupfix == P_DUPFIX);
  couple->outer.checking = (CHECKS2_ENABLED() || (txn->env->flags & MDBX_VALIDATION)) ? z_pagecheck | z_leaf : z_leaf;
  couple->outer.subcur = nullptr;

  if (tree->flags & MDBX_DUPSORT) {
    couple->inner.cursor.signature = cur_signature_live;
    subcur_t *const mx = couple->outer.subcur = &couple->inner;
    mx->cursor.subcur = nullptr;
    mx->cursor.next = &mx->cursor;
    mx->cursor.txn = (MDBX_txn *)txn;
    mx->cursor.tree = &mx->nested_tree;
    mx->cursor.clc = ptr_disp(couple->outer.clc, sizeof(clc_t));
    cASSERT0(txn, &mx->cursor.clc->k == &kvx->clc.v);
    mx->cursor.dbi_state = dbi_state;
    mx->cursor.top_and_flags = z_fresh_mark | z_inner;
    STATIC_ASSERT(MDBX_DUPFIXED * 2 == P_DUPFIX);
    mx->cursor.checking = couple->outer.checking + ((tree->flags & MDBX_DUPFIXED) << 1);
  }

  if (unlikely(*dbi_state & DBI_STALE))
    return tbl_refresh_absent2baddbi(couple->outer.txn, cursor_dbi(&couple->outer));

  return tbl_setup_ifneed(txn->env, kvx, tree);
}

__cold int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, kvx_t *const kvx) {
  return couple_init(couple, txn, tree, kvx, txn->dbi_state);
}

int cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi) {
  STATIC_ASSERT(offsetof(cursor_couple_t, outer) == 0);
  int rc = dbi_check(txn, dbi);
  if (likely(rc == MDBX_SUCCESS))
    rc = couple_init(container_of(mc, cursor_couple_t, outer), txn, &txn->dbs[dbi], &txn->env->kvs[dbi],
                     &txn->dbi_state[dbi]);
  return rc;
}

__cold static int unexpected_dupsort(MDBX_cursor *mc) {
  ERROR("unexpected dupsort-page/node for non-dupsort db/cursor (dbi %zu)", cursor_dbi(mc));
  mc->txn->flags |= MDBX_TXN_ERROR;
  be_poor(mc);
  return MDBX_CORRUPTED;
}

int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp) {
  cASSERT0(mc, is_pointed(mc));
  subcur_t *mx = mc->subcur;
  if (!MDBX_DISABLE_VALIDATION && unlikely(mx == nullptr))
    return unexpected_dupsort(mc);

  const uint8_t flags = node_flags(node);
  switch (flags) {
  default:
    bad_page(mp, "invalid node flags %u", flags);
    goto bailout;
  case N_DUP | N_TREE:
    if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) {
      ERROR("invalid nested-db record size (%zu, expect %zu)", node_ds(node), sizeof(tree_t));
      goto bailout;
    }
    memcpy(&mx->nested_tree, node_data(node), sizeof(tree_t));
    const txnid_t pp_txnid = mp->txnid;
    if (!MDBX_DISABLE_VALIDATION && unlikely(mx->nested_tree.mod_txnid > pp_txnid)) {
      ERROR("nested-db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", mx->nested_tree.mod_txnid, pp_txnid);
      goto bailout;
    }
    mx->cursor.top_and_flags = z_fresh_mark | z_inner;
    break;
  case N_DUP:
    if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) <= PAGEHDRSZ)) {
      ERROR("invalid nested-page size %zu", node_ds(node));
      goto bailout;
    }
    page_t *sp = node_data(node);
    mx->nested_tree.height = 1;
    mx->nested_tree.branch_pages = 0;
    mx->nested_tree.leaf_pages = 1;
    mx->nested_tree.large_pages = 0;
    mx->nested_tree.items = page_numkeys(sp);
    mx->nested_tree.root = 0;
    mx->nested_tree.mod_txnid = mp->txnid;
    mx->cursor.top_and_flags = z_inner;
    mx->cursor.pg[0] = sp;
    mx->cursor.ki[0] = 0;
    mx->nested_tree.flags = flags_db2sub(mc->tree->flags);
    mx->nested_tree.dupfix_size = (mc->tree->flags & MDBX_DUPFIXED) ? sp->dupfix_ksize : 0;
    break;
  }

  if (unlikely(mx->nested_tree.dupfix_size != mc->tree->dupfix_size)) {
    if (!MDBX_DISABLE_VALIDATION && unlikely(mc->tree->dupfix_size != 0)) {
      ERROR("cursor mismatched nested-db dupfix_size %u", mc->tree->dupfix_size);
      goto bailout;
    }
    if (!MDBX_DISABLE_VALIDATION && unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0)) {
      ERROR("mismatched nested-db flags %u", mc->tree->flags);
      goto bailout;
    }
    if (!MDBX_DISABLE_VALIDATION &&
        unlikely(mx->nested_tree.dupfix_size < mc->clc->v.lmin || mx->nested_tree.dupfix_size > mc->clc->v.lmax)) {
      ERROR("mismatched nested-db.dupfix_size (%u) <> min/max value-length "
            "(%zu/%zu)",
            mx->nested_tree.dupfix_size, mc->clc->v.lmin, mc->clc->v.lmax);
      goto bailout;
    }
    mc->tree->dupfix_size = mx->nested_tree.dupfix_size;
    mc->clc->v.lmin = mc->clc->v.lmax = mx->nested_tree.dupfix_size;
    cASSERT0(mc, mc->clc->v.lmax >= mc->clc->v.lmin);
  }

  DEBUG("Sub-db dbi -%zu root page %" PRIaPGNO, cursor_dbi(&mx->cursor), mx->nested_tree.root);
  return MDBX_SUCCESS;

bailout:
  mx->cursor.top_and_flags = z_poor_mark | z_inner;
  return MDBX_CORRUPTED;
}

/*----------------------------------------------------------------------------*/

MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc, MDBX_cursor *cdst) {
  cASSERT0(cdst, cdst->txn == csrc->txn);
  cASSERT0(cdst, cdst->clc == csrc->clc);
  cASSERT0(cdst, cdst->dbi_state == csrc->dbi_state);
  cdst->top_and_flags = csrc->top_and_flags;

  for (intptr_t i = 0; i <= csrc->top; i++) {
    cdst->pg[i] = csrc->pg[i];
    cdst->ki[i] = csrc->ki[i];
  }
  return cdst;
}

MDBX_cursor *cursor_copy_position(const MDBX_cursor *csrc, MDBX_cursor *cdst) {
  cursor_cpstk(csrc, cdst);
  if (cdst->subcur) {
    cdst->subcur->cursor.tree->root = 0;
    cdst->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
    if (is_pointed(&csrc->subcur->cursor)) {
      cursor_cpstk(&csrc->subcur->cursor, &cdst->subcur->cursor);
      *cdst->subcur->cursor.tree = *csrc->subcur->cursor.tree;
    }
  }
  return cdst;
}

static inline void cursor_clone_half(const MDBX_cursor *csrc, MDBX_cursor *cdst) {
  cdst->next = nullptr;
  cdst->backup = nullptr;
  cdst->subcur = nullptr;
  cdst->clc = csrc->clc;
  cdst->txn = csrc->txn;
  cdst->dbi_state = csrc->dbi_state;
  cdst->checking = csrc->checking;
  cdst->tree = csrc->tree;
  cursor_cpstk(csrc, cdst);
}

MDBX_cursor *cursor_clone_slightly(const MDBX_cursor *csrc, cursor_couple_t *couple) {
  cASSERT0(csrc, csrc->txn->txnid >= csrc->txn->env->lck->cached_oldest_txnid.weak);
  couple->outer.next = nullptr;
  couple->outer.backup = nullptr;
  couple->outer.subcur = nullptr;
  couple->outer.clc = nullptr;
  couple->outer.txn = csrc->txn;
  couple->outer.dbi_state = csrc->dbi_state;
  couple->outer.checking = z_pagecheck;
  couple->outer.tree = nullptr;
  couple->outer.top_and_flags = 0;

  MDBX_cursor *cdst = &couple->outer;
  if (is_inner(csrc)) {
    couple->inner.cursor.next = nullptr;
    couple->inner.cursor.backup = nullptr;
    couple->inner.cursor.subcur = nullptr;
    couple->inner.cursor.txn = csrc->txn;
    couple->inner.cursor.dbi_state = csrc->dbi_state;
    couple->outer.subcur = &couple->inner;
    cdst = &couple->inner.cursor;
  }

  cdst->checking = csrc->checking;
  cdst->tree = csrc->tree;
  cdst->clc = csrc->clc;

  return cursor_cpstk(csrc, cdst);
}

MDBX_cursor *cursor_clone_complete(const MDBX_cursor *csrc, cursor_couple_t *couple) {
  if (unlikely(is_inner(csrc)))
    return cursor_clone_slightly(csrc, couple);

  cursor_clone_half(csrc, &couple->outer);
  if (csrc->subcur) {
    couple->outer.subcur = &couple->inner;
    cursor_clone_half(&csrc->subcur->cursor, &couple->inner.cursor);
    couple->inner.nested_tree = *couple->inner.cursor.tree;
    couple->inner.cursor.tree = &couple->inner.nested_tree;
  }
  return &couple->outer;
}

static __always_inline int sibling(MDBX_cursor *mc, bool right) {
  if (mc->top < 1) {
    /* root has no siblings */
    return MDBX_NOTFOUND;
  }

  cursor_pop(mc);
  DEBUG("parent page is page %" PRIaPGNO ", index %u", mc->pg[mc->top]->pgno, mc->ki[mc->top]);

  int err;
  if (right ? (mc->ki[mc->top] + (size_t)1 >= page_numkeys(mc->pg[mc->top])) : (mc->ki[mc->top] == 0)) {
    DEBUG("no more keys aside, moving to next %s sibling", right ? "right" : "left");
    err = right ? cursor_sibling_right(mc) : cursor_sibling_left(mc);
    if (unlikely(err != MDBX_SUCCESS)) {
      if (likely(err == MDBX_NOTFOUND)) {
        /* undo cursor_pop() before returning */
        mc->top += 1;
        return err;
      }
      goto bailout;
    }
  } else {
    mc->ki[mc->top] += right ? 1 : -1;
    DEBUG("just moving to %s index key %u", right ? "right" : "left", mc->ki[mc->top]);
  }
  cASSERT0(mc, is_branch(mc->pg[mc->top]));

  page_t *mp = mc->pg[mc->top];
  const node_t *node = page_node(mp, mc->ki[mc->top]);
  err = page_get(mc, node_pgno(node), &mp, mp->txnid);
  if (likely(err == MDBX_SUCCESS)) {
    err = cursor_push(mc, mp, right ? 0 : (indx_t)page_numkeys(mp) - 1);
    if (likely(err == MDBX_SUCCESS))
      return err;
  }

bailout:
  cASSERT0(mc, err != MDBX_NOTFOUND);
  be_poor(mc);
  return err;
}

__hot int cursor_sibling_left(MDBX_cursor *mc) {
  int err = sibling(mc, false);
  if (likely(err != MDBX_NOTFOUND))
    return err;

  cASSERT0(mc, mc->top >= 0);
  size_t nkeys = page_numkeys(mc->pg[mc->top]);
  cASSERT0(mc, nkeys > 0);
  mc->ki[mc->top] = 0;
  /* не очищаем состояние вложенного курсора, а остаёмся на предыдущей позиции */
  return MDBX_NOTFOUND;
}

__hot int cursor_sibling_right(MDBX_cursor *mc) {
  int err = sibling(mc, true);
  if (likely(err != MDBX_NOTFOUND))
    return err;

  cASSERT0(mc, mc->top >= 0);
  size_t nkeys = page_numkeys(mc->pg[mc->top]);
  cASSERT0(mc, nkeys > 0);
  mc->ki[mc->top] = (indx_t)nkeys - 1;
  mc->flags = z_eof_soft | z_eof_hard | (mc->flags & z_clear_mask);
  /* не очищаем состояние вложенного курсора, а остаёмся на предыдущей позиции */
  return MDBX_NOTFOUND;
}

intptr_t cursor_cmp(const MDBX_cursor *left, const MDBX_cursor *right) {
  ASSERT(left->top == right->top);
  intptr_t cmp = 0;
  for (intptr_t i = cmp; i <= left->top; ++i) {
    ASSERT(left->pg[i] == right->pg[i]);
    cmp = (intptr_t)left->ki[i] - (intptr_t)right->ki[i];
    if (cmp)
      break;
  }
  return cmp;
}

/*----------------------------------------------------------------------------*/

/* Функция-шаблон: Приземляет курсор на данные в текущей позиции.
 * В том числе, загружает данные во вложенный курсор при его наличии. */
static __always_inline int cursor_bring(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc,
                                        MDBX_val *__restrict key, MDBX_val *__restrict data, bool eof) {
  if (inner) {
    cASSERT0(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0);
  } else {
    cASSERT0(mc, (mc->flags & z_inner) == 0);
  }

  const page_t *mp = mc->pg[mc->top];
  if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
    ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
    return MDBX_CORRUPTED;
  }

  const size_t nkeys = page_numkeys(mp);
  cASSERT0(mc, nkeys > 0);
  const size_t ki = mc->ki[mc->top];
  cASSERT0(mc, nkeys > ki);
  cASSERT0(mc, !eof || ki == nkeys - 1);

  if (inner && is_dupfix_leaf(mp)) {
    be_filled(mc);
    if (eof)
      mc->flags |= z_eof_soft;
    if (likely(key))
      *key = page_dupfix_key(mp, ki, mc->tree->dupfix_size);
    return MDBX_SUCCESS;
  }

  const node_t *__restrict node = page_node(mp, ki);
  if (!inner && (node_flags(node) & N_DUP)) {
    int err = cursor_dupsort_setup(mc, node, mp);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr);
    if (node_flags(node) & N_TREE) {
      err = tend2first ? inner_first(&mc->subcur->cursor, data) : inner_last(&mc->subcur->cursor, data);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    } else {
      if (!tend2first) {
        mc->subcur->cursor.ki[0] = (indx_t)mc->subcur->nested_tree.items - 1;
        mc->subcur->cursor.flags |= z_eof_soft;
      }
      if (data) {
        const page_t *inner_mp = mc->subcur->cursor.pg[0];
        cASSERT0(mc, is_subpage(inner_mp) && is_leaf(inner_mp));
        const size_t inner_ki = mc->subcur->cursor.ki[0];
        if (is_dupfix_leaf(inner_mp))
          *data = page_dupfix_key(inner_mp, inner_ki, mc->tree->dupfix_size);
        else
          *data = get_key(page_node(inner_mp, inner_ki));
      }
    }
    be_filled(mc);
  } else {
    if (!inner)
      inner_gone(mc);
    if (data) {
      int err = node_read(mc, node, data, mp);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }
    be_filled(mc);
    if (eof)
      mc->flags |= z_eof_soft;
  }

  get_key_optional(node, key);
  return MDBX_SUCCESS;
}

/* Функция-шаблон: Устанавливает курсор в начало или конец. */
static __always_inline int cursor_brim(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc,
                                       MDBX_val *__restrict key, MDBX_val *__restrict data) {
  if (mc->top != 0) {
    int err = tree_search(mc, nullptr, tend2first ? Z_FIRST : Z_LAST);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }
  const size_t nkeys = page_numkeys(mc->pg[mc->top]);
  cASSERT0(mc, nkeys > 0);
  mc->ki[mc->top] = tend2first ? 0 : nkeys - 1;
  return cursor_bring(inner, tend2first, mc, key, data, !tend2first);
}

__hot int inner_first(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, true, mc, data, nullptr); }

__hot int inner_last(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, false, mc, data, nullptr); }

__hot int outer_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
  return cursor_brim(false, true, mc, key, data);
}

__hot int outer_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
  return cursor_brim(false, false, mc, key, data);
}

/*----------------------------------------------------------------------------*/

/* Функция-шаблон: Передвигает курсор на одну позицию.
 * При необходимости управляет вложенным курсором. */
static __always_inline int cursor_step(const bool inner, const bool forward, MDBX_cursor *__restrict mc,
                                       MDBX_val *__restrict key, MDBX_val *__restrict data, MDBX_cursor_op op) {
  if (forward) {
    if (inner)
      cASSERT0(mc, op == MDBX_NEXT);
    else
      cASSERT0(mc, op == MDBX_NEXT || op == MDBX_NEXT_DUP || op == MDBX_NEXT_NODUP);
  } else {
    if (inner)
      cASSERT0(mc, op == MDBX_PREV);
    else
      cASSERT0(mc, op == MDBX_PREV || op == MDBX_PREV_DUP || op == MDBX_PREV_NODUP);
  }
  if (inner) {
    cASSERT0(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0);
  } else {
    cASSERT0(mc, (mc->flags & z_inner) == 0);
  }

  if (unlikely(is_poor(mc))) {
    int state = mc->flags;
    if (state & z_fresh) {
      if (forward)
        return inner ? inner_first(mc, key) : outer_first(mc, key, data);
      else
        return inner ? inner_last(mc, key) : outer_last(mc, key, data);
    }
    mc->flags = inner ? z_inner | z_poor_mark : z_poor_mark;
    return (state & z_after_delete) ? MDBX_NOTFOUND : MDBX_ENODATA;
  }

  const page_t *mp = mc->pg[mc->top];
  const intptr_t nkeys = page_numkeys(mp);
  cASSERT0(mc, nkeys > 0);

  intptr_t ki = mc->ki[mc->top];
  const uint8_t state = mc->flags & (z_after_delete | z_hollow | z_eof_hard | z_eof_soft);
  if (likely(state == 0)) {
    cASSERT0(mc, ki < nkeys);
    if (!inner && op != (forward ? MDBX_NEXT_NODUP : MDBX_PREV_NODUP)) {
      int err = MDBX_NOTFOUND;
      if (inner_pointed(mc)) {
        err = forward ? inner_next(&mc->subcur->cursor, data) : inner_prev(&mc->subcur->cursor, data);
        if (likely(err == MDBX_SUCCESS)) {
          get_key_optional(page_node(mp, ki), key);
          return MDBX_SUCCESS;
        }
        if (unlikely(err != MDBX_NOTFOUND && err != MDBX_ENODATA)) {
          cASSERT0(mc, !inner_pointed(mc));
          return err;
        }
        cASSERT0(mc, !forward || (mc->subcur->cursor.flags & z_eof_soft));
      }
      if (op == (forward ? MDBX_NEXT_DUP : MDBX_PREV_DUP))
        return err;
    }
  } else {
    if (mc->flags & z_hollow) {
      cASSERT0(mc, !inner_pointed(mc) || inner_hollow(mc));
      return MDBX_ENODATA;
    }

    if (!inner && op == (forward ? MDBX_NEXT_DUP : MDBX_PREV_DUP))
      return MDBX_NOTFOUND;

    if (forward) {
      if (state & z_after_delete) {
        if (ki < nkeys)
          goto bring;
      } else {
        cASSERT0(mc, state & (z_eof_soft | z_eof_hard));
        return MDBX_NOTFOUND;
      }
    } else if (state & z_eof_hard) {
      mc->ki[mc->top] = (indx_t)nkeys - 1;
      goto bring;
    }
  }

  DEBUG("turn-%s: top page was %" PRIaPGNO " in cursor %p, ki %zi of %zi", forward ? "next" : "prev", mp->pgno,
        __Wpedantic_format_voidptr(mc), ki, nkeys);
  if (forward) {
    if (likely(++ki < nkeys))
      mc->ki[mc->top] = (indx_t)ki;
    else {
      DEBUG("%s", "=====> move to next sibling page");
      int err = cursor_sibling_right(mc);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      mp = mc->pg[mc->top];
      DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]);
    }
  } else {
    if (likely(--ki >= 0))
      mc->ki[mc->top] = (indx_t)ki;
    else {
      DEBUG("%s", "=====> move to prev sibling page");
      int err = cursor_sibling_left(mc);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      mp = mc->pg[mc->top];
      DEBUG("prev page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]);
    }
  }
  DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u", mp->pgno, page_numkeys(mp),
        mc->ki[mc->top]);

bring:
  return cursor_bring(inner, forward, mc, key, data, false);
}

__hot int inner_next(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, true, mc, data, nullptr, MDBX_NEXT); }

__hot int inner_prev(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, false, mc, data, nullptr, MDBX_PREV); }

__hot int outer_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
  return cursor_step(false, true, mc, key, data, op);
}

__hot int outer_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
  return cursor_step(false, false, mc, key, data, op);
}

/*----------------------------------------------------------------------------*/

__hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) {
  int err;
  DKBUF_DEBUG;
  MDBX_env *const env = mc->txn->env;
  if (LOG_ENABLED(MDBX_LOG_DEBUG) && (flags & MDBX_RESERVE))
    data->iov_base = nullptr;
  DEBUG("==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR, cursor_dbi_dbg(mc), DKEY_DEBUG(key),
        key->iov_len, DVAL_DEBUG(data), data->iov_len);

  if ((flags & MDBX_CURRENT) != 0 && (mc->flags & z_inner) == 0) {
    if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE)))
      return MDBX_EINVAL;
    /* Запрошено обновление текущей записи, на которой сейчас стоит курсор.
     * Проверяем что переданный ключ совпадает со значением в текущей позиции
     * курсора. Здесь проще вызвать cursor_ops(), так как для обслуживания
     * таблиц с MDBX_DUPSORT также требуется текущий размер данных. */
    MDBX_val current_key, current_data;
    err = cursor_ops(mc, &current_key, &current_data, MDBX_GET_CURRENT);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if (mc->clc->k.cmp(key, &current_key) != 0)
      return MDBX_EKEYMISMATCH;

    if (unlikely((flags & MDBX_MULTIPLE))) {
      if (unlikely(!mc->subcur))
        return MDBX_EINVAL;
      err = cursor_del(mc, flags & MDBX_ALLDUPS);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      if (unlikely(data[1].iov_len == 0))
        return MDBX_SUCCESS;
      flags -= MDBX_CURRENT;
      goto skip_check_samedata;
    }

    if (mc->subcur) {
      node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]);
      if (node_flags(node) & N_DUP) {
        cASSERT0(mc, inner_pointed(mc));
        /* Если за ключом более одного значения, либо если размер данных
         * отличается, то вместо обновления требуется удаление и
         * последующая вставка. */
        if (mc->subcur->nested_tree.items > 1 || current_data.iov_len != data->iov_len) {
          err = cursor_del(mc, flags & MDBX_ALLDUPS);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
          flags -= MDBX_CURRENT;
          goto skip_check_samedata;
        }
      } else if (unlikely(node_size(key, data) > env->leaf_nodemax)) {
        /* Уже есть пара key-value хранящаяся в обычном узле. Новые данные
         * слишком большие для размещения в обычном узле вместе с ключом, но
         * могут быть размещены в вложенном дереве. Удаляем узел со старыми
         * данными, чтобы при помещении новых создать вложенное дерево. */
        err = cursor_del(mc, 0);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
        flags -= MDBX_CURRENT;
        goto skip_check_samedata;
      }
    }
    if (!(flags & MDBX_RESERVE) && unlikely(eq_fast(&current_data, data)))
      return MDBX_SUCCESS /* the same data, nothing to update */;
  skip_check_samedata:;
  }

  int rc = MDBX_SUCCESS;
  if (mc->tree->height == 0) {
    /* new database, cursor has nothing to point to */
    cASSERT0(mc, is_poor(mc));
    rc = MDBX_NO_ROOT;
  } else if ((flags & MDBX_CURRENT) == 0) {
    bool exact = false;
    MDBX_val old_data;
    if ((flags & MDBX_APPEND) && mc->tree->items > 0) {
      MDBX_val last_key;
      old_data.iov_base = nullptr;
      old_data.iov_len = 0;
      rc = (mc->flags & z_inner) ? inner_last(mc, &last_key) : outer_last(mc, &last_key, &old_data);
      if (likely(rc == MDBX_SUCCESS)) {
        const int cmp = mc->clc->k.cmp(key, &last_key);
        if (likely(cmp > 0)) {
          mc->ki[mc->top]++; /* step forward for appending */
          rc = MDBX_NOTFOUND;
#ifdef MDBX_EVENBUG20260405_FIX
          /* 2026-04-05: Здесь занятный эффект чётности унаследованных от LMDB багов. Исправлять один без другого
           * нельзя. Однако, оба исправления увеличивают исполнимый код и накладные расходы. Поэтому решено поместить
           * исправление под #ifdef до появления явной необходимости в них. */
          inner_gone(mc);
#endif /* MDBX_EVENBUG20260405_FIX */
        } else if (unlikely(cmp != 0)) {
          /* new-key < last-key */
          return MDBX_EKEYMISMATCH;
        } else {
          rc = MDBX_SUCCESS;
          exact = true;
        }
      }
    } else {
      csr_t csr = cursor_seek(mc, (MDBX_val *)key, &old_data, MDBX_SET);
      rc = csr.err;
      exact = csr.exact;
    }
    if (exact) {
      cASSERT0(mc, rc == MDBX_SUCCESS);
      if (unlikely(flags & MDBX_NOOVERWRITE)) {
        DEBUG("duplicate key [%s]", DKEY_DEBUG(key));
        *data = old_data;
        return MDBX_KEYEXIST;
      }
      if (unlikely(mc->flags & z_inner)) {
        /* nested subtree of DUPSORT-database with the same key, nothing to update */
        return (flags & MDBX_NODUPDATA) ? MDBX_KEYEXIST : MDBX_SUCCESS;
      }
      if (inner_pointed(mc)) {
        if (unlikely(flags & MDBX_ALLDUPS)) {
          rc = cursor_del(mc, MDBX_ALLDUPS);
          if (unlikely(rc != MDBX_SUCCESS))
            return rc;
          flags -= MDBX_ALLDUPS;
          cASSERT0(mc, mc->top + 1 == mc->tree->height);
          rc = (mc->top >= 0) ? MDBX_NOTFOUND : MDBX_NO_ROOT;
        } else if ((flags & (MDBX_RESERVE | MDBX_MULTIPLE)) == 0) {
          old_data = *data;
          csr_t csr = cursor_seek(&mc->subcur->cursor, &old_data, nullptr, MDBX_SET_RANGE);
          if (unlikely(csr.exact)) {
            cASSERT0(mc, csr.err == MDBX_SUCCESS);
            if (flags & MDBX_NODUPDATA)
              return MDBX_KEYEXIST;
            if (flags & MDBX_APPENDDUP)
              return MDBX_EKEYMISMATCH;
            /* the same data, nothing to update */
            return MDBX_SUCCESS;
          } else if (csr.err != MDBX_SUCCESS && unlikely(csr.err != MDBX_NOTFOUND)) {
            be_poor(mc);
            return csr.err;
          }
        }
      } else if (!(flags & (MDBX_RESERVE | MDBX_MULTIPLE))) {
        if (unlikely(eq_fast(data, &old_data))) {
          cASSERT1(mc, mc->clc->v.cmp(data, &old_data) == 0);
          /* the same data, nothing to update */
          return (mc->subcur && (flags & MDBX_NODUPDATA)) ? MDBX_KEYEXIST : MDBX_SUCCESS;
        }
        cASSERT1(mc, mc->clc->v.cmp(data, &old_data) != 0);
      }
    } else if (unlikely(rc != MDBX_NOTFOUND))
      return rc;
  }

  mc->flags &= ~z_after_delete;
  MDBX_val xdata, *ref_data = data;
  size_t *batch_dupfix_done = nullptr, batch_dupfix_given = 0;
  if (unlikely(flags & MDBX_MULTIPLE)) {
    batch_dupfix_given = data[1].iov_len;
    if (unlikely(data[1].iov_len == 0))
      return /* nothing todo */ MDBX_SUCCESS;
    batch_dupfix_done = &data[1].iov_len;
    *batch_dupfix_done = 0;
  }

  /* Cursor is positioned, check for room in the dirty list */
  err = cursor_touch(mc, key, ref_data);
  if (unlikely(err))
    return err;

  if (unlikely(rc == MDBX_NO_ROOT)) {
    /* new database, write a root leaf page */
    DEBUG("%s", "allocating new root leaf page");
    pgr_t npr = page_new(mc, P_LEAF);
    if (unlikely(npr.err != MDBX_SUCCESS))
      return npr.err;
    npr.err = cursor_push(mc, npr.page, 0);
    if (unlikely(npr.err != MDBX_SUCCESS))
      return npr.err;
    mc->tree->root = npr.page->pgno;
    mc->tree->height++;
    if (mc->tree->flags & MDBX_INTEGERKEY) {
      ASSERT(key->iov_len >= mc->clc->k.lmin && key->iov_len <= mc->clc->k.lmax);
      mc->clc->k.lmin = mc->clc->k.lmax = key->iov_len;
    }
    if (mc->tree->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) {
      ASSERT(data->iov_len >= mc->clc->v.lmin && data->iov_len <= mc->clc->v.lmax);
      ASSERT(mc->subcur != nullptr);
      mc->tree->dupfix_size = /* mc->subcur->nested_tree.dupfix_size = */
          (unsigned)(mc->clc->v.lmin = mc->clc->v.lmax = data->iov_len);
      cASSERT0(mc, mc->clc->v.lmin == mc->subcur->cursor.clc->k.lmin);
      cASSERT0(mc, mc->clc->v.lmax == mc->subcur->cursor.clc->k.lmax);
      if (mc->flags & z_inner)
        npr.page->flags |= P_DUPFIX;
    }
  }

  MDBX_val old_singledup, old_data;
  tree_t nested_dupdb;
  page_t *sub_root = nullptr;
  bool insert_key, insert_data;
  uint16_t fp_flags = P_LEAF;
  page_t *fp = env->page_auxbuf;
  fp->txnid = mc->txn->front_txnid;
  insert_key = insert_data = (rc != MDBX_SUCCESS);
  old_singledup.iov_base = nullptr;
  old_singledup.iov_len = 0;
  if (insert_key) {
    /* The key does not exist */
    DEBUG("inserting key at index %i", mc->ki[mc->top]);
    if (mc->tree->flags & MDBX_DUPSORT) {
      inner_gone(mc);
      if (node_size(key, data) > env->leaf_nodemax) {
        /* Too big for a node, insert in sub-DB.  Set up an empty
         * "old sub-page" for convert_to_subtree to expand to a full page. */
        fp->dupfix_ksize = (mc->tree->flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0;
        fp->lower = fp->upper = 0;
        old_data.iov_len = PAGEHDRSZ;
        goto convert_to_subtree;
      }
    }
  } else {
    /* there's only a key anyway, so this is a no-op */
    if (is_dupfix_leaf(mc->pg[mc->top])) {
      size_t ksize = mc->tree->dupfix_size;
      if (unlikely(key->iov_len != ksize))
        return MDBX_BAD_VALSIZE;
      void *ptr = page_dupfix_ptr(mc->pg[mc->top], mc->ki[mc->top], ksize);
      memcpy(ptr, key->iov_base, ksize);
    fix_parent:
      /* if overwriting slot 0 of leaf, need to
       * update branch key if there is a parent page */
      if (mc->top && !mc->ki[mc->top]) {
        size_t dtop = 1;
        mc->top--;
        /* slot 0 is always an empty key, find real slot */
        while (mc->top && !mc->ki[mc->top]) {
          mc->top--;
          dtop++;
        }
        err = MDBX_SUCCESS;
        if (mc->ki[mc->top])
          err = tree_propagate_key(mc, key);
        cASSERT0(mc, mc->top + dtop < UINT16_MAX);
        mc->top += (uint8_t)dtop;
        if (unlikely(err != MDBX_SUCCESS))
          return err;
      }

      if (CHECKS2_ENABLED()) {
        err = cursor_validate(mc);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
      }
      return MDBX_SUCCESS;
    }

  more:
    if (CHECKS2_ENABLED()) {
      err = cursor_validate(mc);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }
    node_t *const node = page_node(mc->pg[mc->top], mc->ki[mc->top]);

    /* Large/Overflow page overwrites need special handling */
    if (unlikely(node_flags(node) & N_BIG)) {
      const size_t dpages = (node_size(key, data) > env->leaf_nodemax) ? largechunk_npages(env, data->iov_len) : 0;

      const pgno_t pgno = node_largedata_pgno(node);
      pgr_t lp = page_get_large(mc, pgno, mc->pg[mc->top]->txnid);
      if (unlikely(lp.err != MDBX_SUCCESS))
        return lp.err;
      cASSERT0(mc, page_type(lp.page) == P_LARGE);

      /* Is the ov page from this txn (or a parent) and big enough? */
      const size_t ovpages = lp.page->pages;
      const size_t extra_threshold =
          (mc->tree == &mc->txn->dbs[FREE_DBI]) ? 1 : /* LY: add configurable threshold to keep reserve space */ 0;
      if (!is_frozen(mc->txn, lp.page) && ovpages >= dpages && ovpages <= dpages + extra_threshold) {
        /* yes, overwrite it. */
        if (!is_modifable(mc->txn, lp.page)) {
          if (is_spilled(mc->txn, lp.page)) {
            lp = /* TODO: avoid search and get txn & spill-index from page_result */
                page_unspill(mc->txn, lp.page);
            if (unlikely(lp.err))
              return lp.err;
          } else {
            if (unlikely(!mc->txn->parent)) {
              ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s "
                    "page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
                    " without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN,
                    "large/overflow", pgno, lp.page->txnid, mc->txn->txnid, mc->txn->front_txnid);
              return MDBX_PROBLEM;
            }

            /* It is writable only in a parent txn */
            page_t *np = page_shadow_alloc(mc->txn, ovpages);
            if (unlikely(!np))
              return MDBX_ENOMEM;

            memcpy(np, lp.page, PAGEHDRSZ); /* Copy header of page */
            err = page_dirty(mc->txn, lp.page = np, ovpages);
            if (unlikely(err != MDBX_SUCCESS))
              return err;
            if (MDBX_ENABLE_PGOP_STAT)
              mc->txn->env->lck->pgops.clone.weak += ovpages;
            cASSERT1(mc, txn_dpl_check(mc->txn));
          }
        }
        node_set_ds(node, data->iov_len);
        if (flags & MDBX_RESERVE)
          data->iov_base = page2payload(lp.page);
        else
          memcpy(page2payload(lp.page), data->iov_base, data->iov_len);

        if (CHECKS2_ENABLED()) {
          err = cursor_validate(mc);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
        }
        return MDBX_SUCCESS;
      }

      if ((err = page_retire(mc, lp.page)) != MDBX_SUCCESS)
        return err;
    } else {
      old_data.iov_len = node_ds(node);
      old_data.iov_base = node_data(node);
      cASSERT0(mc, ptr_disp(old_data.iov_base, old_data.iov_len) <= ptr_disp(mc->pg[mc->top], env->ps));

      /* DB has dups? */
      if (mc->tree->flags & MDBX_DUPSORT) {
        /* Prepare (sub-)page/sub-DB to accept the new item, if needed.
         * fp: old sub-page or a header faking it.
         * mp: new (sub-)page.
         * xdata: node data with new sub-page or sub-DB. */
        size_t growth = 0; /* growth in page size.*/
        page_t *mp = fp = xdata.iov_base = env->page_auxbuf;
        mp->pgno = mc->pg[mc->top]->pgno;

        /* Was a single item before, must convert now */
        if (!(node_flags(node) & N_DUP)) {
          /* does data match? */
          if (flags & MDBX_APPENDDUP) {
            const int cmp = mc->clc->v.cmp(data, &old_data);
            cASSERT1(mc, cmp != 0 || eq_fast(data, &old_data));
            if (unlikely(cmp <= 0))
              return MDBX_EKEYMISMATCH;
          } else if (eq_fast(data, &old_data)) {
            cASSERT1(mc, mc->clc->v.cmp(data, &old_data) == 0);
            cASSERT0(mc, !"Should not happen since" || batch_dupfix_done);
            if (flags & MDBX_NODUPDATA)
              return MDBX_KEYEXIST;
            /* data is match exactly byte-to-byte, nothing to update */
            rc = MDBX_SUCCESS;
            if (unlikely(batch_dupfix_done))
              goto batch_dupfix_continue;
            return rc;
          }

          /* Just overwrite the current item */
          if (flags & MDBX_CURRENT) {
            cASSERT0(mc, node_size(key, data) <= env->leaf_nodemax);
            goto current;
          }

          /* Back up original data item */
          memcpy(old_singledup.iov_base = fp + 1, old_data.iov_base, old_singledup.iov_len = old_data.iov_len);

          /* Make sub-page header for the dup items, with dummy body */
          fp->flags = P_LEAF | P_SUBP;
          fp->lower = 0;
          xdata.iov_len = PAGEHDRSZ + old_data.iov_len + data->iov_len;
          if (mc->tree->flags & MDBX_DUPFIXED) {
            fp->flags |= P_DUPFIX;
            fp->dupfix_ksize = (uint16_t)data->iov_len;
            /* Будем создавать DUPFIX-страницу, как минимум с двумя элементами.
             * При коротких значениях и наличии свободного места можно сделать
             * некоторое резервирование места, чтобы при последующих добавлениях
             * не сразу расширять созданную под-страницу.
             * Резервирование в целом сомнительно (см ниже), но может сработать
             * в плюс (а если в минус то несущественный) при коротких ключах. */
            xdata.iov_len +=
                page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len, data->iov_len);
            cASSERT0(mc, (xdata.iov_len & 1) == 0);
          } else {
            xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) + (old_data.iov_len & 1) + (data->iov_len & 1);
          }
          cASSERT0(mc, (xdata.iov_len & 1) == 0);
          fp->upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ);
          old_data.iov_len = xdata.iov_len; /* pretend olddata is fp */
        } else if (node_flags(node) & N_TREE) {
          /* Data is on sub-DB, just store it */
          flags |= N_DUP | N_TREE;
          goto dupsort_put;
        } else {
          /* Data is on sub-page */
          fp = old_data.iov_base;
          switch (flags) {
          default:
            growth = is_dupfix_leaf(fp) ? fp->dupfix_ksize : (node_size(data, nullptr) + sizeof(indx_t));
            if (page_room(fp) >= growth) {
              /* На текущей под-странице есть место для добавления элемента.
               * Оптимальнее продолжить использовать эту страницу, ибо
               * добавление вложенного дерева увеличит WAF на одну страницу. */
              goto continue_subpage;
            }
            /* На текущей под-странице нет места для еще одного элемента.
             * Можно либо увеличить эту под-страницу, либо вынести куст
             * значений во вложенное дерево.
             *
             * Продолжать использовать текущую под-страницу возможно
             * только пока и если размер после добавления элемента будет
             * меньше leaf_nodemax. Соответственно, при превышении
             * просто сразу переходим на вложенное дерево. */
            xdata.iov_len = old_data.iov_len + (growth += growth & 1);
            if (xdata.iov_len > env->subpage_limit)
              goto convert_to_subtree;

            /* Можно либо увеличить под-страницу, в том числе с некоторым
             * запасом, либо перейти на вложенное поддерево.
             *
             * Резервирование места на под-странице представляется сомнительным:
             *  - Резервирование увеличит рыхлость страниц, в том числе
             *    вероятность разделения основной/гнездовой страницы;
             *  - Сложно предсказать полезный размер резервирования,
             *    особенно для не-MDBX_DUPFIXED;
             *  - Наличие резерва позволяет съекономить только на перемещении
             *    части элементов основной/гнездовой страницы при последующих
             *    добавлениях в нее элементов. Причем после первого изменения
             *    размера под-страницы, её тело будет примыкать
             *    к неиспользуемому месту на основной/гнездовой странице,
             *    поэтому последующие последовательные добавления потребуют
             *    только передвижения в entries[].
             *
             * Соответственно, более важным/определяющим представляется
             * своевременный переход к вложеному дереву, но тут достаточно
             * сложный конфликт интересов:
             *  - При склонности к переходу к вложенным деревьям, суммарно
             *    в БД будет большее кол-во более рыхлых страниц. Это увеличит
             *    WAF, а также RAF при последовательных чтениях большой БД.
             *    Однако, при коротких ключах и большом кол-ве
             *    дубликатов/мультизначений, плотность ключей в листовых
             *    страницах основного дерева будет выше. Соответственно, будет
             *    пропорционально меньше branch-страниц. Поэтому будет выше
             *    вероятность оседания/не-вымывания страниц основного дерева из
             *    LRU-кэша, а также попадания в write-back кэш при записи.
             *  - Наоботот, при склонности к использованию под-страниц, будут
             *    наблюдаться обратные эффекты. Плюс некоторые накладные расходы
             *    на лишнее копирование данных под-страниц в сценариях
             *    нескольких обонвлений дубликатов одного куста в одной
             *    транзакции.
             *
             * Суммарно наиболее рациональным представляется такая тактика:
             *  - Вводим три порога subpage_limit, subpage_room_threshold
             *    и subpage_reserve_prereq, которые могут быть
             *    заданы/скорректированы пользователем в ‰ от leaf_nodemax;
             *  - Используем под-страницу пока её размер меньше subpage_limit
             *    и на основной/гнездовой странице не-менее
             *    subpage_room_threshold свободного места;
             *  - Резервируем место только для 1-3 коротких dupfix-элементов,
             *    расширяя размер под-страницы на размер кэш-линии ЦПУ, но
             *    только если на странице не менее subpage_reserve_prereq
             *    свободного места.
             *  - По-умолчанию устанавливаем:
             *     subpage_limit = leaf_nodemax (1000‰);
             *     subpage_room_threshold = 0;
             *     subpage_reserve_prereq = leaf_nodemax (1000‰).
             */
            if (is_dupfix_leaf(fp))
              growth += page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len,
                                              data->iov_len);
            else {
              /* TODO: Если добавить возможность для пользователя задавать
               * min/max размеров ключей/данных, то здесь разумно реализовать
               * тактику резервирования подобную dupfixed. */
            }
            break;

          case MDBX_CURRENT | MDBX_NODUPDATA:
          case MDBX_CURRENT:
          continue_subpage:
            fp->txnid = mc->txn->front_txnid;
            fp->pgno = mp->pgno;
            mc->subcur->cursor.pg[0] = fp;
            flags |= N_DUP;
            goto dupsort_put;
          }
          xdata.iov_len = old_data.iov_len + growth;
          cASSERT0(mc, (xdata.iov_len & 1) == 0);
        }

        fp_flags = fp->flags;
        if (xdata.iov_len > env->subpage_limit || node_size_len(node_ks(node), xdata.iov_len) > env->leaf_nodemax ||
            (env->subpage_room_threshold &&
             page_room(mc->pg[mc->top]) + node_size_len(node_ks(node), old_data.iov_len) <
                 env->subpage_room_threshold + node_size_len(node_ks(node), xdata.iov_len))) {
          /* Too big for a sub-page, convert to sub-DB */
        convert_to_subtree:
          fp_flags &= ~P_SUBP;
          nested_dupdb.dupfix_size = 0;
          nested_dupdb.flags = flags_db2sub(mc->tree->flags);
          if (mc->tree->flags & MDBX_DUPFIXED) {
            fp_flags |= P_DUPFIX;
            nested_dupdb.dupfix_size = fp->dupfix_ksize;
          }
          nested_dupdb.height = 1;
          nested_dupdb.branch_pages = 0;
          nested_dupdb.leaf_pages = 1;
          nested_dupdb.large_pages = 0;
          nested_dupdb.items = page_numkeys(fp);
          xdata.iov_len = sizeof(nested_dupdb);
          xdata.iov_base = &nested_dupdb;
          const pgr_t par = gc_alloc_single(mc);
          mp = par.page;
          if (unlikely(par.err != MDBX_SUCCESS))
            return par.err;
          mc->tree->leaf_pages += 1;
          cASSERT0(mc, env->ps > old_data.iov_len);
          growth = env->ps - (unsigned)old_data.iov_len;
          cASSERT0(mc, (growth & 1) == 0);
          flags |= N_DUP | N_TREE;
          nested_dupdb.root = mp->pgno;
          nested_dupdb.sequence = 0;
          nested_dupdb.mod_txnid = mc->txn->txnid;
          sub_root = mp;
        }
        if (mp != fp) {
          mp->flags = fp_flags;
          mp->txnid = mc->txn->front_txnid;
          mp->dupfix_ksize = fp->dupfix_ksize;
          mp->lower = fp->lower;
          cASSERT0(mc, fp->upper + growth < UINT16_MAX);
          mp->upper = fp->upper + (indx_t)growth;
          if (unlikely(fp_flags & P_DUPFIX)) {
            memcpy(page2payload(mp), page2payload(fp), page_numkeys(fp) * fp->dupfix_ksize);
            cASSERT0(mc, (((mp->dupfix_ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0);
          } else {
            cASSERT0(mc, (mp->upper & 1) == 0);
            memcpy(ptr_disp(mp, mp->upper + PAGEHDRSZ), ptr_disp(fp, fp->upper + PAGEHDRSZ),
                   old_data.iov_len - fp->upper - PAGEHDRSZ);
            memcpy(mp->entries, fp->entries, page_numkeys(fp) * sizeof(mp->entries[0]));
            for (size_t i = 0; i < page_numkeys(fp); i++) {
              cASSERT0(mc, mp->entries[i] + growth <= UINT16_MAX);
              mp->entries[i] += (indx_t)growth;
            }
          }
        }

        if (!insert_key)
          node_del(mc, 0);
        ref_data = &xdata;
        flags |= N_DUP;
        goto insert_node;
      }

      /* MDBX passes N_TREE in 'flags' to write a DB record */
      if (unlikely((node_flags(node) ^ flags) & N_TREE))
        return MDBX_INCOMPATIBLE;

    current:
      if (data->iov_len == old_data.iov_len) {
        cASSERT0(mc, EVEN_CEIL(key->iov_len) == EVEN_CEIL(node_ks(node)));
        /* same size, just replace it. Note that we could
         * also reuse this node if the new data is smaller,
         * but instead we opt to shrink the node in that case. */
        if (flags & MDBX_RESERVE)
          data->iov_base = old_data.iov_base;
        else if (!(mc->flags & z_inner))
          memcpy(old_data.iov_base, data->iov_base, data->iov_len);
        else {
          cASSERT0(mc, page_numkeys(mc->pg[mc->top]) == 1);
          cASSERT0(mc, page_type_compat(mc->pg[mc->top]) == P_LEAF);
          cASSERT0(mc, node_ds(node) == 0);
          cASSERT0(mc, node_flags(node) == 0);
          cASSERT0(mc, key->iov_len < UINT16_MAX);
          node_set_ks(node, key->iov_len);
          memcpy(node_key(node), key->iov_base, key->iov_len);
          cASSERT0(mc, ptr_disp(node_key(node), node_ds(node)) < ptr_disp(mc->pg[mc->top], env->ps));
          goto fix_parent;
        }

        if (CHECKS2_ENABLED()) {
          err = cursor_validate(mc);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
        }
        return MDBX_SUCCESS;
      }
    }
    node_del(mc, 0);
  }

  ref_data = data;

insert_node:;
  const unsigned naf = flags & NODE_ADD_FLAGS;
  size_t nsize = is_dupfix_leaf(mc->pg[mc->top]) ? key->iov_len : leaf_size(env, key, ref_data);
  if (page_room(mc->pg[mc->top]) < nsize) {
    rc = page_split(mc, key, ref_data, P_INVALID, insert_key ? naf : naf | MDBX_SPLIT_REPLACE);
    if (rc == MDBX_SUCCESS && CHECKS2_ENABLED())
      rc = insert_key ? cursor_validate(mc) : cursor_validate_updating(mc);
  } else {
    /* There is room already in this leaf page. */
    if (is_dupfix_leaf(mc->pg[mc->top])) {
      cASSERT0(mc, !(naf & (N_BIG | N_TREE | N_DUP)) && ref_data->iov_len == 0);
      rc = node_add_dupfix(mc, mc->ki[mc->top], key);
    } else
      rc = node_add_leaf(mc, mc->ki[mc->top], key, ref_data, naf);
    if (likely(rc == 0)) {
      /* Adjust other cursors pointing to mp */
      page_t *const mp = mc->pg[mc->top];
      const size_t dbi = cursor_dbi(mc);
      for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) {
        MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
        if (!is_related(mc, m3) || m3->pg[mc->top] != mp)
          continue;
        if (m3->ki[mc->top] >= mc->ki[mc->top])
          m3->ki[mc->top] += insert_key;
        if (inner_pointed(m3))
          cursor_inner_refresh(m3, mp, m3->ki[mc->top]);
      }
    }
  }

  if (likely(rc == MDBX_SUCCESS)) {
    /* Now store the actual data in the child DB. Note that we're
     * storing the user data in the keys field, so there are strict
     * size limits on dupdata. The actual data fields of the child
     * DB are all zero size. */
    if (flags & N_DUP) {
      MDBX_val empty;
    dupsort_put:
      empty.iov_len = 0;
      empty.iov_base = nullptr;
      node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]);
#define SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE 1
      STATIC_ASSERT((MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) == MDBX_NOOVERWRITE);
      unsigned inner_flags = MDBX_CURRENT | ((flags & MDBX_NODUPDATA) >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE);
      if ((flags & MDBX_CURRENT) == 0) {
        inner_flags -= MDBX_CURRENT;
        rc = cursor_dupsort_setup(mc, node, mc->pg[mc->top]);
        if (unlikely(rc != MDBX_SUCCESS))
          goto dupsort_error;
      }
      subcur_t *const mx = mc->subcur;
      if (sub_root) {
        cASSERT0(mc, mx->nested_tree.height == 1 && mx->nested_tree.root == sub_root->pgno);
        mx->cursor.flags = z_inner;
        mx->cursor.top = 0;
        mx->cursor.pg[0] = sub_root;
        mx->cursor.ki[0] = 0;
      }
      if (old_singledup.iov_base) {
        /* converted, write the original data first */
        if (is_dupfix_leaf(mx->cursor.pg[0]))
          rc = node_add_dupfix(&mx->cursor, 0, &old_singledup);
        else
          rc = node_add_leaf(&mx->cursor, 0, &old_singledup, &empty, 0);
        if (unlikely(rc != MDBX_SUCCESS))
          goto dupsort_error;
        mx->cursor.tree->items = 1;
      }
      if (!(node_flags(node) & N_TREE) || sub_root) {
        page_t *const mp = mc->pg[mc->top];
        const intptr_t nkeys = page_numkeys(mp);
        const size_t dbi = cursor_dbi(mc);

        for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) {
          if (!is_related(mc, m2) || m2->pg[mc->top] != mp)
            continue;
          if (/* пропускаем незаполненные курсоры, иначе получится что у такого
                 курсора будет инициализирован вложенный, что антилогично и бесполезно. */
              is_filled(m2) && m2->ki[mc->top] == mc->ki[mc->top]) {
            cASSERT0(m2, m2->subcur->cursor.clc == mx->cursor.clc);
            m2->subcur->nested_tree = mx->nested_tree;
            m2->subcur->cursor.pg[0] = mx->cursor.pg[0];
            if (old_singledup.iov_base) {
              m2->subcur->cursor.top_and_flags = z_inner;
              m2->subcur->cursor.ki[0] = 0;
            }
            DEBUG("Sub-dbi -%zu root page %" PRIaPGNO, cursor_dbi(&m2->subcur->cursor), m2->subcur->nested_tree.root);
          } else if (!insert_key && m2->ki[mc->top] < nkeys)
            cursor_inner_refresh(m2, mp, m2->ki[mc->top]);
        }
      }
      cASSERT0(mc, mc->subcur->nested_tree.items < PTRDIFF_MAX);
      const size_t probe = (size_t)mc->subcur->nested_tree.items;
#define SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND 1
      STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) == MDBX_APPEND);
      inner_flags |= (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND;
      rc = cursor_put(&mc->subcur->cursor, data, &empty, inner_flags);
      if (flags & N_TREE) {
        void *db = node_data(node);
        mc->subcur->nested_tree.mod_txnid = mc->txn->txnid;
        memcpy(db, &mc->subcur->nested_tree, sizeof(tree_t));
      }
      insert_data = (probe != (size_t)mc->subcur->nested_tree.items);
    }
    /* Increment count unless we just replaced an existing item. */
    if (insert_data)
      mc->tree->items++;
    if (insert_key) {
      if (unlikely(rc != MDBX_SUCCESS))
        goto dupsort_error;
      /* If we succeeded and the key didn't exist before,
       * make sure the cursor is marked valid. */
      be_filled(mc);
    }
    if (likely(rc == MDBX_SUCCESS)) {
      cASSERT0(mc, is_filled(mc));
      if (unlikely(batch_dupfix_done)) {
      batch_dupfix_continue:
        /* let caller know how many succeeded, if any */
        if ((*batch_dupfix_done += 1) < batch_dupfix_given) {
          data[0].iov_base = ptr_disp(data[0].iov_base, data[0].iov_len);
          insert_key = insert_data = false;
          old_singledup.iov_base = nullptr;
          sub_root = nullptr;
          goto more;
        }
      }
      if (CHECKS2_ENABLED())
        rc = cursor_validate(mc);
    }
    return rc;

  dupsort_error:
    if (unlikely(rc == MDBX_KEYEXIST)) {
      /* should not happen, we deleted that item */
      ERROR("Unexpected %i error while put to nested dupsort's hive", rc);
      rc = MDBX_PROBLEM;
    }
  }
  mc->txn->flags |= MDBX_TXN_ERROR;
  return rc;
}

int cursor_check_multiple(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) {
  (void)key;
  if (unlikely(flags & MDBX_RESERVE))
    return MDBX_EINVAL;
  if (unlikely(!(mc->tree->flags & MDBX_DUPFIXED)))
    return MDBX_INCOMPATIBLE;
  const size_t number = data[1].iov_len;
  if (unlikely(number > MAX_MAPSIZE / 2 / (BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) - NODESIZE))) {
    /* checking for multiplication overflow */
    if (unlikely(number > MAX_MAPSIZE / 2 / data->iov_len))
      return MDBX_TOO_LARGE;
  }
  return MDBX_SUCCESS;
}

__hot int cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) {
  cASSERT0(mc, (mc->flags & z_inner) == 0);
  if (unlikely(key->iov_len > mc->clc->k.lmax || key->iov_len < mc->clc->k.lmin))
    return MDBX_BAD_VALSIZE;
  if (unlikely(data->iov_len > mc->clc->v.lmax || data->iov_len < mc->clc->v.lmin))
    return MDBX_BAD_VALSIZE;

  uint64_t aligned_keybytes, aligned_databytes;
  MDBX_val aligned_key, aligned_data;
  if (mc->tree->flags & MDBX_INTEGERKEY) {
    if (key->iov_len == 8) {
      if (unlikely(7 & (uintptr_t)key->iov_base)) {
        /* copy instead of return error to avoid break compatibility */
        aligned_key.iov_base = bcopy_8(&aligned_keybytes, key->iov_base);
        aligned_key.iov_len = key->iov_len;
        key = &aligned_key;
      }
    } else if (key->iov_len == 4) {
      if (unlikely(3 & (uintptr_t)key->iov_base)) {
        /* copy instead of return error to avoid break compatibility */
        aligned_key.iov_base = bcopy_4(&aligned_keybytes, key->iov_base);
        aligned_key.iov_len = key->iov_len;
        key = &aligned_key;
      }
    } else
      return MDBX_BAD_VALSIZE;

    if (unlikely(mc->clc->k.lmin != mc->clc->k.lmax))
      mc->clc->k.lmin = mc->clc->k.lmax = key->iov_len;
  }
  if (mc->tree->flags & MDBX_INTEGERDUP) {
    if (data->iov_len == 8) {
      if (unlikely(7 & (uintptr_t)data->iov_base)) {
        if (unlikely(flags & MDBX_MULTIPLE)) {
          /* LY: использование alignof(uint64_t) тут не подходил из-за ошибок
           * MSVC и некоторых других компиляторов, когда для элементов
           * массивов/векторов обеспечивает выравнивание только на 4-х байтовых
           * границу и одновременно alignof(uint64_t) == 8. */
          if (MDBX_WORDBITS > 32 || (3 & (uintptr_t)data->iov_base) != 0)
            return MDBX_BAD_VALSIZE;
        } else {
          /* copy instead of return error to avoid break compatibility */
          aligned_data.iov_base = bcopy_8(&aligned_databytes, data->iov_base);
          aligned_data.iov_len = data->iov_len;
          data = &aligned_data;
        }
      }
    } else if (data->iov_len == 4) {
      if (unlikely(3 & (uintptr_t)data->iov_base)) {
        if (unlikely(flags & MDBX_MULTIPLE))
          return MDBX_BAD_VALSIZE;
        /* copy instead of return error to avoid break compatibility */
        aligned_data.iov_base = bcopy_4(&aligned_databytes, data->iov_base);
        aligned_data.iov_len = data->iov_len;
        data = &aligned_data;
      }
    } else
      return MDBX_BAD_VALSIZE;
    if (unlikely(mc->clc->v.lmin != mc->clc->v.lmax))
      mc->clc->v.lmin = mc->clc->v.lmax = data->iov_len;
  }
  return cursor_put(mc, key, data, flags);
}

__hot int cursor_del(MDBX_cursor *mc, unsigned flags) {
  if (unlikely(!is_filled(mc))) {
    if (!F_ISSET(mc->flags, z_inner | z_eof_hard))
      return MDBX_ENODATA;
    if (outer_next(cursor_outer(mc), nullptr, nullptr, MDBX_NEXT) != MDBX_SUCCESS)
      return MDBX_ENODATA;
  }

  int rc = cursor_touch(mc, nullptr, nullptr);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  page_t *mp = mc->pg[mc->top];
  cASSERT0(mc, is_modifable(mc->txn, mp));
  if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
    ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
    return MDBX_CORRUPTED;
  }
  if (is_dupfix_leaf(mp))
    goto del_key;

  node_t *node = page_node(mp, mc->ki[mc->top]);
  if (node_flags(node) & N_DUP) {
    if (flags & (MDBX_ALLDUPS | /* for compatibility */ MDBX_NODUPDATA)) {
      /* will subtract the final entry later */
      mc->tree->items -= mc->subcur->nested_tree.items - 1;
    } else {
      if ((node_flags(node) & N_TREE) == 0) {
        page_t *sp = node_data(node);
        cASSERT0(mc, is_subpage(sp));
        sp->txnid = mp->txnid;
        mc->subcur->cursor.pg[0] = sp;
      }
      rc = cursor_del(&mc->subcur->cursor, 0);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      /* If sub-DB still has entries, we're done */
      if (mc->subcur->nested_tree.items) {
        if (node_flags(node) & N_TREE) {
          /* update table info */
          mc->subcur->nested_tree.mod_txnid = mc->txn->txnid;
          memcpy(node_data(node), &mc->subcur->nested_tree, sizeof(tree_t));
          /* fix other sub-DB cursors pointed at the same sub-tree */
          for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
            if (is_related(mc, m2) && m2->pg[mc->top] == mp && m2->ki[mc->top] == mc->ki[mc->top])
              m2->subcur->nested_tree = mc->subcur->nested_tree;
          }
        } else {
          /* shrink sub-page */
          node = node_shrink(mp, mc->ki[mc->top], node);
          mc->subcur->cursor.pg[0] = node_data(node);
          /* fix other sub-DB cursors pointed at sub-pages on this page */
          for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
            if (!is_related(mc, m2) || m2->pg[mc->top] != mp)
              continue;
            const node_t *inner = node;
            if (unlikely(m2->ki[mc->top] >= page_numkeys(mp))) {
              m2->flags |= z_eof_hard | z_eof_soft | z_after_delete;
              m2->subcur->nested_tree.root = 0;
              m2->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
              continue;
            }
            if (m2->ki[mc->top] != mc->ki[mc->top]) {
              inner = page_node(mp, m2->ki[mc->top]);
              if (node_flags(inner) != N_DUP)
                continue;
            } else
              m2->subcur->nested_tree = mc->subcur->nested_tree;
            m2->subcur->cursor.pg[0] = node_data(inner);
          }
        }
        mc->tree->items -= 1;
        cASSERT0(mc, mc->tree->items > 0 && mc->tree->height > 0 && mc->tree->root != P_INVALID);
        return rc;
      }
      /* otherwise fall thru and delete the sub-DB */
    }

    if ((node_flags(node) & N_TREE) && mc->subcur->cursor.tree->height) {
      /* add all the child DB's pages to the free list */
      rc = tree_drop(&mc->subcur->cursor);
      if (unlikely(rc != MDBX_SUCCESS))
        goto fail;
    }
    inner_gone(mc);
  } else {
    cASSERT0(mc, !inner_pointed(mc));
    /* MDBX passes N_TREE in 'flags' to delete a DB record */
    if (unlikely((node_flags(node) ^ flags) & N_TREE))
      return MDBX_INCOMPATIBLE;
  }

  /* add large/overflow pages to free list */
  if (node_flags(node) & N_BIG) {
    pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid);
    if (unlikely((rc = lp.err) || (rc = page_retire(mc, lp.page))))
      goto fail;
  }

del_key:
  mc->tree->items -= 1;
  const MDBX_dbi dbi = cursor_dbi(mc);
  indx_t ki = mc->ki[mc->top];
  mp = mc->pg[mc->top];
  cASSERT0(mc, is_leaf(mp));
  node_del(mc, mc->tree->dupfix_size);

  /* Adjust other cursors pointing to mp */
  for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) {
    MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
    if (!is_related(mc, m3) || m3->pg[mc->top] != mp)
      continue;
    if (m3->ki[mc->top] == ki) {
      m3->flags |= z_after_delete;
      inner_gone(m3);
    } else {
      m3->ki[mc->top] -= m3->ki[mc->top] > ki;
      if (inner_pointed(m3))
        cursor_inner_refresh(m3, mp, m3->ki[mc->top]);
    }
  }

  rc = tree_rebalance(mc);
  if (unlikely(rc != MDBX_SUCCESS))
    goto fail;

  mc->flags |= z_after_delete;
  inner_gone(mc);
  if (unlikely(mc->top < 0)) {
    /* DB is totally empty now, just bail out.
     * Other cursors adjustments were already done
     * by rebalance and aren't needed here. */
    cASSERT0(mc, mc->tree->items == 0 && (mc->tree->root == P_INVALID || (is_inner(mc) && !mc->tree->root)) &&
                     mc->flags < 0);
    return MDBX_SUCCESS;
  }

  ki = mc->ki[mc->top];
  mp = mc->pg[mc->top];
  cASSERT0(mc, is_leaf(mc->pg[mc->top]));
  size_t nkeys = page_numkeys(mp);
  cASSERT0(mc, (mc->tree->items > 0 && nkeys > 0) || ((mc->flags & z_inner) && mc->tree->items == 0 && nkeys == 0));

  /* Adjust this and other cursors pointing to mp */
  const intptr_t top = /* может быть сброшен в -1 */ mc->top;
  for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) {
    MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
    if (top > m3->top || m3->pg[top] != mp)
      continue;
    /* if m3 points past last node in page, find next sibling */
    if (m3->ki[top] >= nkeys) {
      rc = cursor_sibling_right(m3);
      if (rc == MDBX_NOTFOUND) {
        rc = MDBX_SUCCESS;
        continue;
      }
      if (unlikely(rc != MDBX_SUCCESS))
        goto fail;
    }
    if (/* пропускаем незаполненные курсоры, иначе получится что у такого
           курсора будет инициализирован вложенный, что алогично и бесполезно. */
        is_filled(m3) && m3->subcur &&
        (m3->ki[top] >= ki ||
         /* уже переместились вправо */ m3->pg[top] != mp)) {
      node = page_node(m3->pg[m3->top], m3->ki[m3->top]);
      /* Если это dupsort-узел, то должен быть валидный вложенный курсор. */
      if (node_flags(node) & N_DUP) {
        /* Тут три варианта событий:
         * 1) Вложенный курсор уже инициализирован, у узла есть флаг N_TREE,
         *    соответственно дубликаты вынесены в отдельное дерево с корнем
         *    в отдельной странице = ничего корректировать не требуется.
         * 2) Вложенный курсор уже инициализирован, у узла нет флага N_TREE,
         *    соответственно дубликаты размещены на вложенной sub-странице.
         * 3) Курсор стоял на удалённом элементе, который имел одно значение,
         *    а после удаления переместился на следующий элемент с дубликатами.
         *    В этом случае вложенный курсор не инициализирован и теперь его
         *    нужно установить на первый дубликат. */
        if (is_pointed(&m3->subcur->cursor)) {
          if ((node_flags(node) & N_TREE) == 0) {
            cASSERT0(m3, m3->subcur->cursor.top == 0 && m3->subcur->nested_tree.height == 1);
            m3->subcur->cursor.pg[0] = node_data(node);
          }
        } else {
          rc = cursor_dupsort_setup(m3, node, m3->pg[m3->top]);
          if (unlikely(rc != MDBX_SUCCESS))
            goto fail;
          if (node_flags(node) & N_TREE) {
            rc = inner_first(&m3->subcur->cursor, nullptr);
            if (unlikely(rc != MDBX_SUCCESS))
              goto fail;
          }
        }
      } else
        inner_gone(m3);
    }
  }

  cASSERT0(mc, rc == MDBX_SUCCESS);
  if (CHECKS2_ENABLED())
    rc = cursor_validate(mc);
  return rc;

fail:
  mc->txn->flags |= MDBX_TXN_ERROR;
  return rc;
}

/*----------------------------------------------------------------------------*/

__hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
  DKBUF_DEBUG;

  alignkey_t aligned;
  csr_t ret = {.exact = false, .err = check_key(mc, key, &aligned)};
  if (unlikely(ret.err != MDBX_SUCCESS))
    return ret;

  page_t *mp;
  node_t *node = nullptr;
  /* See if we're already on the right page */
  if (is_pointed(mc)) {
    mp = mc->pg[mc->top];
    cASSERT0(mc, is_leaf(mp));
    const size_t nkeys = page_numkeys(mp);
    if (unlikely(nkeys == 0)) {
      /* при создании первой листовой страницы */
      cASSERT0(mc, mc->top == 0 && mc->tree->height == 1 && mc->tree->branch_pages == 0 && mc->tree->leaf_pages == 1 &&
                       mc->ki[0] == 0);
      /* Логически верно, но нет смысла, ибо это мимолетная/временная
       * ситуация до добавления элемента выше по стеку вызовов:
         mc->flags |= z_eof_soft | z_hollow; */
      ret.err = MDBX_NOTFOUND;
      return ret;
    }

    MDBX_val nodekey;
    if (is_dupfix_leaf(mp))
      nodekey = page_dupfix_key(mp, 0, mc->tree->dupfix_size);
    else {
      node = page_node(mp, 0);
      nodekey = get_key(node);
      inner_gone(mc);
    }
    int cmp = mc->clc->k.cmp(&aligned.key, &nodekey);
    if (unlikely(cmp == 0)) {
      /* Probably happens rarely, but first node on the page was the one we wanted. */
      mc->ki[mc->top] = 0;
      ret.exact = true;
      goto got_node;
    }

    if (cmp > 0) {
      /* Искомый ключ больше первого на этой странице,
       * целевая позиция на этой странице либо правее (ближе к концу). */
      if (likely(nkeys > 1)) {
        if (is_dupfix_leaf(mp)) {
          nodekey.iov_base = page_dupfix_ptr(mp, nkeys - 1, nodekey.iov_len);
        } else {
          node = page_node(mp, nkeys - 1);
          nodekey = get_key(node);
        }
        cmp = mc->clc->k.cmp(&aligned.key, &nodekey);
        if (cmp == 0) {
          /* last node was the one we wanted */
          mc->ki[mc->top] = (indx_t)(nkeys - 1);
          ret.exact = true;
          goto got_node;
        }
        if (cmp < 0) {
          /* Искомый ключ между первым и последним на этой страницы,
           * поэтому пропускаем поиск по дереву и продолжаем только на текущей
           * странице. */
          /* Сравниваем с текущей позицией, ибо частным сценарием является такое
           * совпадение, но не делаем проверку если текущая позиция является
           * первой/последний и соответственно такое сравнение было выше. */
          if (mc->ki[mc->top] > 0 && mc->ki[mc->top] < nkeys - 1) {
            if (is_dupfix_leaf(mp)) {
              nodekey.iov_base = page_dupfix_ptr(mp, mc->ki[mc->top], nodekey.iov_len);
            } else {
              node = page_node(mp, mc->ki[mc->top]);
              nodekey = get_key(node);
            }
            cmp = mc->clc->k.cmp(&aligned.key, &nodekey);
            if (cmp == 0) {
              /* current node was the one we wanted */
              ret.exact = true;
              goto got_node;
            }
          }
          goto search_node;
        }
      }

      /* Если в стеке курсора есть страницы справа, то продолжим искать там. */
      cASSERT0(mc, mc->tree->height > mc->top);
      for (intptr_t i = 0; i < mc->top; i++)
        if ((size_t)mc->ki[i] + 1 < page_numkeys(mc->pg[i]))
          goto continue_other_pages;

      /* Ключ больше последнего. */
      mc->ki[mc->top] = (indx_t)nkeys;
      if (op < MDBX_SET_RANGE) {
      target_not_found:
        cASSERT0(mc, op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE);
        /* Операция предполагает поиск конкретного ключа, который не найден.
         * Поэтому переводим курсор в неустановленное состояние, но без сброса
         * top, что позволяет работать fastpath при последующем поиске по дереву
         * страниц. */
        mc->flags |= z_hollow;
        if (inner_pointed(mc))
          mc->subcur->cursor.flags |= z_hollow;
        ret.err = MDBX_NOTFOUND;
        return ret;
      }
      cASSERT0(mc, op == MDBX_SET_RANGE);
      mc->flags = z_eof_soft | z_eof_hard | (mc->flags & z_clear_mask);
      ret.err = MDBX_NOTFOUND;
      return ret;
    }

    if (mc->top == 0) {
      /* There are no other pages */
      mc->ki[mc->top] = 0;
      if (op >= MDBX_SET_RANGE)
        goto got_node;
      else
        goto target_not_found;
    }
  }
  cASSERT0(mc, !inner_pointed(mc));

continue_other_pages:
  ret.err = tree_search(mc, &aligned.key, 0);
  if (unlikely(ret.err != MDBX_SUCCESS))
    return ret;

  cASSERT0(mc, is_pointed(mc) && !inner_pointed(mc));
  mp = mc->pg[mc->top];
  MDBX_ANALYSIS_ASSUME(mp != nullptr);
  cASSERT0(mc, is_leaf(mp));

search_node:
  cASSERT0(mc, is_pointed(mc) && !inner_pointed(mc));
  sfr_t sr = tree_search_foliage(mc, &aligned.key);
  node = sr.node;
  ret.exact = sr.exact;
  if (!ret.exact) {
    if (op < MDBX_SET_RANGE)
      goto target_not_found;

    if (node == nullptr) {
      DEBUG("%s", "===> inexact leaf not found, goto sibling");
      ret.err = cursor_sibling_right(mc);
      if (unlikely(ret.err != MDBX_SUCCESS))
        return ret; /* no entries matched */
      mp = mc->pg[mc->top];
      cASSERT0(mc, is_leaf(mp));
      if (!is_dupfix_leaf(mp))
        node = page_node(mp, 0);
    }
  }

got_node:
  cASSERT0(mc, is_pointed(mc) && !inner_pointed(mc));
  cASSERT0(mc, mc->ki[mc->top] < page_numkeys(mc->pg[mc->top]));
  if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
    ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
    ret.err = MDBX_CORRUPTED;
    return ret;
  }

  if (is_dupfix_leaf(mp)) {
    if (op >= MDBX_SET_KEY)
      *key = page_dupfix_key(mp, mc->ki[mc->top], mc->tree->dupfix_size);
    be_filled(mc);
    ret.err = MDBX_SUCCESS;
    return ret;
  }

  if (node_flags(node) & N_DUP) {
    ret.err = cursor_dupsort_setup(mc, node, mp);
    if (unlikely(ret.err != MDBX_SUCCESS))
      return ret;
    if (op >= MDBX_SET) {
      MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr);
      if (node_flags(node) & N_TREE) {
        ret.err = inner_first(&mc->subcur->cursor, data);
        if (unlikely(ret.err != MDBX_SUCCESS))
          return ret;
      } else if (data) {
        const page_t *inner_mp = mc->subcur->cursor.pg[0];
        cASSERT0(mc, is_subpage(inner_mp) && is_leaf(inner_mp));
        const size_t inner_ki = mc->subcur->cursor.ki[0];
        if (is_dupfix_leaf(inner_mp))
          *data = page_dupfix_key(inner_mp, inner_ki, mc->tree->dupfix_size);
        else
          *data = get_key(page_node(inner_mp, inner_ki));
      }
    } else {
      MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr);
      ret = cursor_seek(&mc->subcur->cursor, data, nullptr, MDBX_SET_RANGE);
      if (unlikely(ret.err != MDBX_SUCCESS)) {
        if (ret.err == MDBX_NOTFOUND && op < MDBX_SET_RANGE)
          goto target_not_found;
        return ret;
      }
      if (op == MDBX_GET_BOTH && !ret.exact)
        goto target_not_found;
    }
  } else if (likely(data)) {
    if (op <= MDBX_GET_BOTH_RANGE) {
      if (unlikely(data->iov_len < mc->clc->v.lmin || data->iov_len > mc->clc->v.lmax)) {
        ret.err = MDBX_BAD_VALSIZE;
        return ret;
      }
      MDBX_val aligned_data = *data;
      uint64_t aligned_databytes;
      if (mc->tree->flags & MDBX_INTEGERDUP) {
        if (aligned_data.iov_len == 8) {
          if (unlikely(7 & (uintptr_t)aligned_data.iov_base))
            /* copy instead of return error to avoid break compatibility */
            aligned_data.iov_base = bcopy_8(&aligned_databytes, aligned_data.iov_base);
        } else if (aligned_data.iov_len == 4) {
          if (unlikely(3 & (uintptr_t)aligned_data.iov_base))
            /* copy instead of return error to avoid break compatibility */
            aligned_data.iov_base = bcopy_4(&aligned_databytes, aligned_data.iov_base);
        } else {
          ret.err = MDBX_BAD_VALSIZE;
          return ret;
        }
      }
      MDBX_val actual_data;
      ret.err = node_read(mc, node, &actual_data, mc->pg[mc->top]);
      if (unlikely(ret.err != MDBX_SUCCESS))
        return ret;
      const int cmp = mc->clc->v.cmp(&aligned_data, &actual_data);
      if (cmp) {
        if (op != MDBX_GET_BOTH_RANGE) {
          cASSERT0(mc, op == MDBX_GET_BOTH);
          goto target_not_found;
        }
        if (cmp > 0) {
          ret.err = MDBX_NOTFOUND;
          return ret;
        }
      }
      *data = actual_data;
    } else {
      ret.err = node_read(mc, node, data, mc->pg[mc->top]);
      if (unlikely(ret.err != MDBX_SUCCESS))
        return ret;
    }
  }

  /* The key already matches in all other cases */
  if (op >= MDBX_SET_KEY)
    get_key_optional(node, key);

  DEBUG("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key), DVAL_DEBUG(data));
  ret.err = MDBX_SUCCESS;
  be_filled(mc);
  return ret;
}

__hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, const MDBX_cursor_op op) {
  if (op != MDBX_GET_CURRENT)
    DEBUG(">> cursor %p(0x%x), ops %u, key %p, value %p", __Wpedantic_format_voidptr(mc), mc->flags, op,
          __Wpedantic_format_voidptr(key), __Wpedantic_format_voidptr(data));
  int rc;

  switch (op) {
  case MDBX_GET_CURRENT:
    cASSERT0(mc, (mc->flags & z_inner) == 0);
    if (unlikely(!is_filled(mc))) {
      if (is_hollow(mc))
        return MDBX_ENODATA;
      if (mc->ki[mc->top] >= page_numkeys(mc->pg[mc->top]))
        return MDBX_NOTFOUND;
    }
    if (mc->flags & z_after_delete)
      return outer_next(mc, key, data, MDBX_NEXT_NODUP);
    else if (inner_pointed(mc) && (mc->subcur->cursor.flags & z_after_delete))
      return outer_next(mc, key, data, MDBX_NEXT_DUP);
    else {
      const page_t *mp = mc->pg[mc->top];
      const node_t *node = page_node(mp, mc->ki[mc->top]);
      get_key_optional(node, key);
      if (!data)
        return MDBX_SUCCESS;
      if (node_flags(node) & N_DUP) {
        if (!MDBX_DISABLE_VALIDATION && unlikely(!mc->subcur))
          return unexpected_dupsort(mc);
        mc = &mc->subcur->cursor;
        if (unlikely(!is_filled(mc))) {
          if (is_hollow(mc))
            return MDBX_ENODATA;
          if (mc->ki[mc->top] >= page_numkeys(mc->pg[mc->top]))
            return MDBX_NOTFOUND;
        }
        mp = mc->pg[mc->top];
        if (is_dupfix_leaf(mp))
          *data = page_dupfix_key(mp, mc->ki[mc->top], mc->tree->dupfix_size);
        else
          *data = get_key(page_node(mp, mc->ki[mc->top]));
        return MDBX_SUCCESS;
      } else {
        cASSERT0(mc, !inner_pointed(mc));
        return node_read(mc, node, data, mc->pg[mc->top]);
      }
    }

  case MDBX_GET_BOTH:
  case MDBX_GET_BOTH_RANGE:
    if (unlikely(data == nullptr))
      return MDBX_EINVAL;
    if (unlikely(mc->subcur == nullptr))
      return MDBX_INCOMPATIBLE;
    /* fall through */
    __fallthrough;
  case MDBX_SET:
  case MDBX_SET_KEY:
  case MDBX_SET_RANGE:
    if (unlikely(key == nullptr))
      return MDBX_EINVAL;
    rc = cursor_seek(mc, key, data, op).err;
    if (rc == MDBX_SUCCESS)
      cASSERT0(mc, is_filled(mc));
    else if (rc == MDBX_NOTFOUND && mc->tree->items) {
      cASSERT0(mc, is_pointed(mc));
      cASSERT0(mc, op == MDBX_SET_RANGE || op == MDBX_GET_BOTH_RANGE || is_hollow(mc));
      cASSERT0(mc, op == MDBX_GET_BOTH_RANGE || inner_hollow(mc));
    } else
      cASSERT0(mc, is_poor(mc) && !is_filled(mc));
    return rc;

  case MDBX_SEEK_AND_GET_MULTIPLE:
    if (unlikely(!key))
      return MDBX_EINVAL;
    rc = cursor_seek(mc, key, data, MDBX_SET).err;
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    __fallthrough /* fall through */;
  case MDBX_GET_MULTIPLE:
    if (unlikely(!data))
      return MDBX_EINVAL;
    if (unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0))
      return MDBX_INCOMPATIBLE;
    if (unlikely(!is_filled(mc)))
      return MDBX_ENODATA;
    if (key) {
      const page_t *mp = mc->pg[mc->top];
      const node_t *node = page_node(mp, mc->ki[mc->top]);
      *key = get_key(node);
    }
    cASSERT0(mc, is_filled(mc));
    if (unlikely(!inner_filled(mc))) {
      if (inner_pointed(mc))
        return MDBX_ENODATA;
      const page_t *mp = mc->pg[mc->top];
      const node_t *node = page_node(mp, mc->ki[mc->top]);
      return node_read(mc, node, data, mp);
    }
    goto fetch_multiple;

  case MDBX_NEXT_MULTIPLE:
    if (unlikely(!data))
      return MDBX_EINVAL;
    if (unlikely(mc->subcur == nullptr))
      return MDBX_INCOMPATIBLE;
    rc = outer_next(mc, key, data, MDBX_NEXT_DUP);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    else {
    fetch_multiple:
      cASSERT0(mc, is_filled(mc) && inner_filled(mc));
      MDBX_cursor *mx = &mc->subcur->cursor;
      data->iov_len = page_numkeys(mx->pg[mx->top]) * mx->tree->dupfix_size;
      data->iov_base = page2payload(mx->pg[mx->top]);
      mx->ki[mx->top] = (indx_t)page_numkeys(mx->pg[mx->top]) - 1;
      return MDBX_SUCCESS;
    }

  case MDBX_PREV_MULTIPLE:
    if (unlikely(!data))
      return MDBX_EINVAL;
    if (unlikely(mc->subcur == nullptr))
      return MDBX_INCOMPATIBLE;
    if (unlikely(!is_filled(mc) || !inner_filled(mc)))
      return MDBX_ENODATA;
    rc = cursor_sibling_left(&mc->subcur->cursor);
    if (likely(rc == MDBX_SUCCESS))
      goto fetch_multiple;
    return rc;

  case MDBX_NEXT_DUP:
  case MDBX_NEXT:
  case MDBX_NEXT_NODUP:
    rc = outer_next(mc, key, data, op);
    mc->flags &= ~z_eof_hard;
    ((cursor_couple_t *)mc)->inner.cursor.flags &= ~z_eof_hard;
    return rc;

  case MDBX_PREV_DUP:
  case MDBX_PREV:
  case MDBX_PREV_NODUP:
    return outer_prev(mc, key, data, op);

  case MDBX_FIRST:
    return outer_first(mc, key, data);
  case MDBX_LAST:
    return outer_last(mc, key, data);

  case MDBX_LAST_DUP:
  case MDBX_FIRST_DUP:
    if (unlikely(data == nullptr))
      return MDBX_EINVAL;
    if (unlikely(!is_filled(mc)))
      return MDBX_ENODATA;
    else {
      node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]);
      get_key_optional(node, key);
      if ((node_flags(node) & N_DUP) == 0)
        return node_read(mc, node, data, mc->pg[mc->top]);
      else if (MDBX_DISABLE_VALIDATION || likely(mc->subcur))
        return ((op == MDBX_FIRST_DUP) ? inner_first : inner_last)(&mc->subcur->cursor, data);
      else
        return unexpected_dupsort(mc);
    }
    break;

  case MDBX_SET_UPPERBOUND:
  case MDBX_SET_LOWERBOUND:
    if (unlikely(key == nullptr || data == nullptr))
      return MDBX_EINVAL;
    else {
      MDBX_val save_data = *data;
      csr_t csr = cursor_seek(mc, key, data, MDBX_SET_RANGE);
      rc = csr.err;
      if (rc == MDBX_SUCCESS && csr.exact && mc->subcur) {
        csr.exact = false;
        if (!save_data.iov_base) {
          /* Avoiding search nested dupfix hive if no data provided.
           * This is changes the semantic of MDBX_SET_LOWERBOUND but avoid
           * returning MDBX_BAD_VALSIZE. */
        } else if (is_pointed(&mc->subcur->cursor)) {
          *data = save_data;
          csr = cursor_seek(&mc->subcur->cursor, data, nullptr, MDBX_SET_RANGE);
          rc = csr.err;
          if (rc == MDBX_NOTFOUND) {
            cASSERT0(mc, !csr.exact);
            rc = outer_next(mc, key, data, MDBX_NEXT_NODUP);
          }
        } else {
          int cmp = mc->clc->v.cmp(&save_data, data);
          csr.exact = (cmp == 0);
          if (cmp > 0)
            rc = outer_next(mc, key, data, MDBX_NEXT_NODUP);
        }
      }
      if (rc == MDBX_SUCCESS && !csr.exact)
        rc = MDBX_RESULT_TRUE;
      if (unlikely(op == MDBX_SET_UPPERBOUND)) {
        /* minor fixups for MDBX_SET_UPPERBOUND */
        if (rc == MDBX_RESULT_TRUE)
          /* already at great-than by MDBX_SET_LOWERBOUND */
          rc = MDBX_SUCCESS;
        else if (rc == MDBX_SUCCESS)
          /* exactly match, going next */
          rc = outer_next(mc, key, data, MDBX_NEXT);
      }
    }
    return rc;

  /* Doubtless API to positioning of the cursor at a specified key. */
  case MDBX_TO_KEY_LESSER_THAN:
  case MDBX_TO_KEY_LESSER_OR_EQUAL:
  case MDBX_TO_KEY_EQUAL:
  case MDBX_TO_KEY_GREATER_OR_EQUAL:
  case MDBX_TO_KEY_GREATER_THAN:
    if (unlikely(key == nullptr))
      return MDBX_EINVAL;
    else {
      csr_t csr = cursor_seek(mc, key, data, MDBX_SET_RANGE);
      rc = csr.err;
      if (csr.exact) {
        cASSERT0(mc, csr.err == MDBX_SUCCESS);
        if (op == MDBX_TO_KEY_LESSER_THAN)
          rc = outer_prev(mc, key, data, MDBX_PREV_NODUP);
        else if (op == MDBX_TO_KEY_GREATER_THAN)
          rc = outer_next(mc, key, data, MDBX_NEXT_NODUP);
      } else if (op < MDBX_TO_KEY_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
        rc = outer_prev(mc, key, data, MDBX_PREV_NODUP);
      else if (op == MDBX_TO_KEY_EQUAL && rc == MDBX_SUCCESS)
        rc = MDBX_NOTFOUND;
    }
    return rc;

  /* Doubtless API to positioning of the cursor at a specified key-value pair
   * for multi-value hives. */
  case MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN:
  case MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL:
  case MDBX_TO_EXACT_KEY_VALUE_EQUAL:
  case MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL:
  case MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN:
    if (unlikely(key == nullptr || data == nullptr))
      return MDBX_EINVAL;
    else {
      MDBX_val save_data = *data;
      csr_t csr = cursor_seek(mc, key, data, MDBX_SET_KEY);
      rc = csr.err;
      if (rc == MDBX_SUCCESS) {
        cASSERT0(mc, csr.exact);
        if (inner_pointed(mc)) {
          MDBX_cursor *const mx = &mc->subcur->cursor;
          csr = cursor_seek(mx, &save_data, nullptr, MDBX_SET_RANGE);
          rc = csr.err;
          if (csr.exact) {
            cASSERT0(mc, csr.err == MDBX_SUCCESS);
            if (op == MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN)
              rc = inner_prev(mx, data);
            else if (op == MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN)
              rc = inner_next(mx, data);
          } else if (op < MDBX_TO_EXACT_KEY_VALUE_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
            rc = inner_prev(mx, data);
          else if (op == MDBX_TO_EXACT_KEY_VALUE_EQUAL && rc == MDBX_SUCCESS)
            rc = MDBX_NOTFOUND;
        } else {
          int cmp = mc->clc->v.cmp(data, &save_data);
          switch (op) {
          default:
            __unreachable();
          case MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN:
            rc = (cmp < 0) ? MDBX_SUCCESS : MDBX_NOTFOUND;
            break;
          case MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL:
            rc = (cmp <= 0) ? MDBX_SUCCESS : MDBX_NOTFOUND;
            break;
          case MDBX_TO_EXACT_KEY_VALUE_EQUAL:
            rc = (cmp == 0) ? MDBX_SUCCESS : MDBX_NOTFOUND;
            break;
          case MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL:
            rc = (cmp >= 0) ? MDBX_SUCCESS : MDBX_NOTFOUND;
            break;
          case MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN:
            rc = (cmp > 0) ? MDBX_SUCCESS : MDBX_NOTFOUND;
            break;
          }
        }
      }
    }
    return rc;

  case MDBX_TO_PAIR_LESSER_THAN:
  case MDBX_TO_PAIR_LESSER_OR_EQUAL:
  case MDBX_TO_PAIR_EQUAL:
  case MDBX_TO_PAIR_GREATER_OR_EQUAL:
  case MDBX_TO_PAIR_GREATER_THAN:
    if (unlikely(key == nullptr || data == nullptr))
      return MDBX_EINVAL;
    else {
      MDBX_val save_data = *data;
      csr_t csr = cursor_seek(mc, key, data, MDBX_SET_RANGE);
      rc = csr.err;
      if (csr.exact) {
        cASSERT0(mc, csr.err == MDBX_SUCCESS);
        if (inner_pointed(mc)) {
          MDBX_cursor *const mx = &mc->subcur->cursor;
          csr = cursor_seek(mx, &save_data, nullptr, MDBX_SET_RANGE);
          rc = csr.err;
          if (csr.exact) {
            cASSERT0(mc, csr.err == MDBX_SUCCESS);
            if (op == MDBX_TO_PAIR_LESSER_THAN)
              rc = outer_prev(mc, key, data, MDBX_PREV);
            else if (op == MDBX_TO_PAIR_GREATER_THAN)
              rc = outer_next(mc, key, data, MDBX_NEXT);
          } else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
            rc = outer_prev(mc, key, data, MDBX_PREV);
          else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS)
            rc = MDBX_NOTFOUND;
          else if (op > MDBX_TO_PAIR_EQUAL && rc == MDBX_NOTFOUND)
            rc = outer_next(mc, key, data, MDBX_NEXT);
        } else {
          int cmp = mc->clc->v.cmp(data, &save_data);
          switch (op) {
          default:
            __unreachable();
          case MDBX_TO_PAIR_LESSER_THAN:
            if (cmp >= 0)
              rc = outer_prev(mc, key, data, MDBX_PREV);
            break;
          case MDBX_TO_PAIR_LESSER_OR_EQUAL:
            if (cmp > 0)
              rc = outer_prev(mc, key, data, MDBX_PREV);
            break;
          case MDBX_TO_PAIR_EQUAL:
            rc = (cmp == 0) ? MDBX_SUCCESS : MDBX_NOTFOUND;
            break;
          case MDBX_TO_PAIR_GREATER_OR_EQUAL:
            if (cmp < 0)
              rc = outer_next(mc, key, data, MDBX_NEXT);
            break;
          case MDBX_TO_PAIR_GREATER_THAN:
            if (cmp <= 0)
              rc = outer_next(mc, key, data, MDBX_NEXT);
            break;
          }
        }
      } else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
        rc = outer_prev(mc, key, data, MDBX_PREV_NODUP);
      else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS)
        rc = MDBX_NOTFOUND;
    }
    return rc;

  default:
    DEBUG("unhandled/unimplemented cursor operation %u", op);
    return MDBX_EINVAL;
  }
}

int cursor_check(const MDBX_cursor *mc, int txn_bad_bits) {
  if (unlikely(mc == nullptr))
    return MDBX_EINVAL;

  if (unlikely(mc->signature != cur_signature_live)) {
    if (mc->signature != cur_signature_ready4dispose)
      return MDBX_EBADSIGN;
    return (txn_bad_bits > MDBX_TXN_FINISHED) ? MDBX_EINVAL : MDBX_SUCCESS;
  }

#if !defined(__SANITIZE_ADDRESS__)
  /* Проверяем что курсор в связанном списке для отслеживания.
   * Исключение допускается только для read-only операций для служебных/временных курсоров на стеке.
   * Но при включении ASAN и LTO компилятор может выносить размещение курсоров со стека очень далеко,
   * из-за чего эта проверка может ложно срабатывать.
   * Например, при вызове mdbx_estimate_distance() из mdbx_estimate_range(). */
  MDBX_MAYBE_UNUSED char stack_top[sizeof(void *)];
  ENSURE_OBJ(mc, cursor_is_tracked(mc) || (!(txn_bad_bits & txn_ro_flat) && stack_top < (char *)mc &&
                                           (char *)mc - stack_top < (ptrdiff_t)globals.sys_pagesize * 4));
#endif /* __SANITIZE_ADDRESS__ */

  if (txn_bad_bits) {
    int rc = check_txn(mc->txn, txn_bad_bits & ~MDBX_TXN_HAS_CHILD);
    if (unlikely(rc != MDBX_SUCCESS)) {
      cASSERT0(mc, rc != MDBX_RESULT_TRUE);
      return rc;
    }

    if (likely((mc->txn->flags & MDBX_TXN_HAS_CHILD) == 0))
      /* связанная с курсором транзакций не имеет дочерних, курсор не требует затенения (возможно уже затенён) */
      return likely(!cursor_dbi_changed(mc)) ? MDBX_SUCCESS : MDBX_BAD_DBI;

    /* связанная с курсором транзакция имеет дочернюю, курсор необходимо затенить, либо DBI-хендл не валиден */
    cASSERT0(mc, (mc->txn->flags & txn_ro_flat) == 0 && mc->txn != mc->txn->env->txn && mc->txn->env->txn);
    rc = dbi_check(mc->txn->env->txn, cursor_dbi(mc));
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;

    /* В результате dbi_check() курсор должен быть затенён */
    cASSERT0(mc, (mc->txn->flags & txn_ro_flat) == 0 && mc->txn == mc->txn->env->txn);
    if (unlikely(mc->txn != mc->txn->env->txn))
      return MDBX_BAD_TXN;
  }

  return MDBX_SUCCESS;
}

int cursor_on_first(const MDBX_cursor *mc) {
  for (intptr_t i = 0; i <= mc->top; ++i) {
    if (mc->ki[i])
      return MDBX_RESULT_FALSE;
  }
  return MDBX_RESULT_TRUE;
}

int cursor_on_last(const MDBX_cursor *mc) {
  for (intptr_t i = 0; i <= mc->top; ++i) {
    size_t nkeys = page_numkeys(mc->pg[i]);
    if (mc->ki[i] < nkeys - 1)
      return MDBX_RESULT_FALSE;
  }
  return MDBX_RESULT_TRUE;
}
//-------------------------------------------------------------------------------------------------

static cdr_t cursor_distance_at(MDBX_cursor *iter, const MDBX_cursor *end, int level) {
  cdr_t tdr = {.err = MDBX_SUCCESS, .distance = 0};
  if (level < iter->top)
    iter->top = level;
  ASSERT(end->top >= iter->top);
  const int iter_ki = iter->ki[iter->top] + ((iter->flags & z_eof_hard) != 0 && level + 1 == iter->tree->height);
  const int end_ki = end->ki[iter->top] + ((end->flags & z_eof_hard) != 0 && level + 1 == iter->tree->height);
  if (iter->pg[iter->top] == end->pg[iter->top])
    tdr.distance = end_ki - iter_ki;
  else {
    tdr.distance = page_numkeys(iter->pg[iter->top]) - iter_ki;
    while (true) {
      tdr.err = cursor_sibling_right(iter);
      if (likely(tdr.err != MDBX_SUCCESS))
        return tdr;
      if (iter->pg[iter->top] == end->pg[iter->top])
        break;
      tdr.distance += page_numkeys(iter->pg[iter->top]);
    }
    tdr.distance += end_ki + ((end->flags & z_eof_hard) != 0 && level + 1 == end->tree->height);
  }
  return tdr;
}

static cdr_t cursor_leaf_amount(const page_t *mp, intptr_t begin, intptr_t end) {
  cdr_t tdr = {.err = MDBX_SUCCESS, .distance = end - begin};
  intptr_t ki = begin;
  do {
    const node_t *node = page_node(mp, ki);
    const size_t dsize = node_ds(node);
    if (node->flags & N_TREE) {
      if (!MDBX_DISABLE_VALIDATION && unlikely(dsize < sizeof(tree_t))) {
        tdr.err = bad_page(mp, "invalid %s-db record size (%zu, expect %zu)\n", "nested", dsize, sizeof(tree_t));
        return tdr;
      }
      tree_t subtree;
      memcpy(&subtree, node_data(node), sizeof(tree_t));
      tdr.distance += subtree.items - 1;
    } else if (node->flags & N_DUP) {
      if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) <= PAGEHDRSZ)) {
        tdr.err = bad_page(mp, "invalid nested/sub-page record size (%zu, expect %u)\n", dsize, PAGEHDRSZ);
        return tdr;
      }
      tdr.distance += page_numkeys(node_data(node)) - 1;
    }
  } while (++ki < end);
  return tdr;
}

cdr_t cursor_distance(MDBX_cursor *iter, const MDBX_cursor *end, int level) {
  ASSERT(iter->tree == end->tree);
  cdr_t tdr = {.err = MDBX_SUCCESS, .distance = 0};
  const intptr_t outer_diff_level = tree_diff_level(iter, end);
  if (unlikely(outer_diff_level < MDBX_RESULT_TRUE)) {
  beyond_end:
    tdr.err = ENODATA;
    return tdr;
  }

  if (level < outer_diff_level)
    return tdr;

  if (!iter->subcur || level < iter->tree->height)
    return cursor_distance_at(iter, end, level);

  if (outer_diff_level == MDBX_RESULT_TRUE) {
    const int cmp = (iter->flags & z_eof_hard) - (end->flags & z_eof_hard);
    if (unlikely(cmp > 0))
      goto beyond_end;
    if (cmp == 0 && inner_pointed(iter)) {
      const intptr_t inner_diff_level = tree_diff_level(&iter->subcur->cursor, &end->subcur->cursor);
      if (unlikely(inner_diff_level < MDBX_RESULT_TRUE))
        goto beyond_end;
    }
  }

  /* запрошенный уровень предполагает учет mutivalues/дубликатов */
  while (iter->pg[iter->top] != end->pg[iter->top]) {
    cdr_t leaf_tdr = cursor_leaf_amount(iter->pg[iter->top], iter->ki[iter->top], page_numkeys(iter->pg[iter->top]));
    tdr.distance += leaf_tdr.distance;
    tdr.err = leaf_tdr.err;
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
    tdr.err = cursor_sibling_right(iter);
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
  }

  if (iter->ki[iter->top] < end->ki[iter->top]) {
    cdr_t leaf_tdr = cursor_leaf_amount(iter->pg[iter->top], iter->ki[iter->top], end->ki[iter->top]);
    tdr.distance += leaf_tdr.distance;
    tdr.err = leaf_tdr.err;
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
  }

  if (is_pointed(&end->subcur->cursor)) {
    iter->pg[iter->top] = end->pg[iter->top];
    iter->ki[iter->top] = end->ki[iter->top];
    tdr.err = cursor_dupsort_setup(iter, page_node(iter->pg[iter->top], iter->ki[iter->top]), iter->pg[iter->top]);
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
    tdr.err = inner_first(&iter->subcur->cursor, nullptr);
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;

    cdr_t subtree_tdr = cursor_distance_at(&iter->subcur->cursor, &end->subcur->cursor, level - iter->tree->height);
    tdr.distance += subtree_tdr.distance;
    tdr.err = subtree_tdr.err;
  }

  return tdr;
}

static cdr_t cursor_forward_at(MDBX_cursor *iter, size_t amount, int level) {
  if (level < iter->top)
    iter->top = level;

  cdr_t tdr = {.err = MDBX_SUCCESS, .distance = amount};
  size_t step = page_numkeys(iter->pg[iter->top]) - iter->ki[iter->top];
  while (tdr.distance >= step) {
    tdr.distance -= step;
    tdr.err = cursor_sibling_right(iter);
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
    step = page_numkeys(iter->pg[iter->top]);
  }

  ASSERT(tdr.distance < step);
  iter->ki[iter->top] += tdr.distance;
  tdr.distance = 0;
  if (iter->top + 1 < iter->tree->height) {
    tdr.err = tree_deepen_edge(iter, Z_FIRST);
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
  }
  if (iter->subcur) {
    iter->subcur->nested_tree.root = 0;
    iter->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
    const node_t *node = page_node(iter->pg[iter->top], iter->ki[iter->top]);
    if ((node_flags(node) & N_DUP)) {
      tdr.err = cursor_dupsort_setup(iter, node, iter->pg[iter->top]);
      if (unlikely(tdr.err != MDBX_SUCCESS))
        return tdr;
      tdr.err = inner_first(&iter->subcur->cursor, nullptr);
      if (unlikely(tdr.err != MDBX_SUCCESS))
        return tdr;
    }
  }

  be_filled(iter);
  return tdr;
}

static cdr_t cursor_backward_at(MDBX_cursor *iter, size_t amount, int level) {
  if (level < iter->top)
    iter->top = level;

  cdr_t tdr = {.err = MDBX_SUCCESS, .distance = amount};
  while (tdr.distance > iter->ki[iter->top]) {
    tdr.distance -= iter->ki[iter->top] + 1;
    tdr.err = cursor_sibling_left(iter);
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
  }

  ASSERT(tdr.distance <= iter->ki[iter->top]);
  iter->ki[iter->top] -= tdr.distance;
  tdr.distance = 0;
  if (iter->top + 1 < iter->tree->height) {
    tdr.err = tree_deepen_edge(iter, Z_LAST);
    if (unlikely(tdr.err != MDBX_SUCCESS))
      return tdr;
  }
  if (iter->subcur) {
    iter->subcur->nested_tree.root = 0;
    iter->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
    const node_t *node = page_node(iter->pg[iter->top], iter->ki[iter->top]);
    if ((node_flags(node) & N_DUP)) {
      tdr.err = cursor_dupsort_setup(iter, node, iter->pg[iter->top]);
      if (unlikely(tdr.err != MDBX_SUCCESS))
        return tdr;
      tdr.err = inner_last(&iter->subcur->cursor, nullptr);
      if (unlikely(tdr.err != MDBX_SUCCESS))
        return tdr;
    }
  }

  be_filled(iter);
  return tdr;
}

int cursor_scroll_forward(MDBX_cursor *mc, intptr_t amount, int level) {
  ASSERT(amount > 0);
  if (level < mc->tree->height || !mc->subcur)
    return cursor_forward_at(mc, amount, level).err;

  /* запрошенный уровень предполагает учет mutivalues/дубликатов */
  int err = MDBX_SUCCESS;
  if (inner_pointed(mc)) {
    cdr_t tdr = cursor_forward_at(&mc->subcur->cursor, amount, level - mc->tree->height);
    err = tdr.err;
    amount = tdr.distance;
    if (unlikely(err != MDBX_NOTFOUND))
      return err;
    mc->subcur->nested_tree.root = 0;
    mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
  }

  while (true) {
    if (++mc->ki[mc->top] >= page_numkeys(mc->pg[mc->top])) {
      err = cursor_sibling_right(mc);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }

    const node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]);
    if ((node->flags & N_DUP) == 0) {
      if (unlikely(--amount == 0))
        return tree_deepen_edge(mc, Z_FIRST);
      continue;
    }

    err = cursor_dupsort_setup(mc, node, mc->pg[mc->top]);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if ((size_t)amount < mc->subcur->nested_tree.items) {
      err = inner_first(&mc->subcur->cursor, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      return cursor_forward_at(&mc->subcur->cursor, amount, level - mc->tree->height).err;
    }
    amount -= mc->subcur->nested_tree.items;
    mc->subcur->nested_tree.root = 0;
    mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
  }
}

int cursor_scroll_backward(MDBX_cursor *mc, intptr_t amount, int level) {
  ASSERT(amount > 0);
  if (level < mc->tree->height || !mc->subcur)
    return cursor_backward_at(mc, amount, level).err;

  /* запрошенный уровень предполагает учет mutivalues/дубликатов */
  int err = MDBX_SUCCESS;
  if (inner_pointed(mc)) {
    cdr_t tdr = cursor_backward_at(&mc->subcur->cursor, amount, level - mc->tree->height);
    err = tdr.err;
    amount = tdr.distance;
    if (unlikely(err != MDBX_NOTFOUND))
      return err;
    mc->subcur->nested_tree.root = 0;
    mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
  }

  while (true) {
    if (mc->ki[mc->top]-- < 1) {
      err = cursor_sibling_left(mc);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }

    const node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]);
    if ((node->flags & N_DUP) == 0) {
      if (unlikely(--amount == 0))
        return tree_deepen_edge(mc, Z_LAST);
      continue;
    }

    err = cursor_dupsort_setup(mc, node, mc->pg[mc->top]);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if ((size_t)amount < mc->subcur->nested_tree.items) {
      err = inner_last(&mc->subcur->cursor, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      return cursor_backward_at(&mc->subcur->cursor, amount, level - mc->tree->height).err;
    }
    amount -= mc->subcur->nested_tree.items;
    mc->subcur->nested_tree.root = 0;
    mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
  }
}

MDBX_MAYBE_UNUSED static intptr_t debug_cursor_distance(const MDBX_cursor *begin, const MDBX_cursor *end,
                                                        unsigned deepness) {
  intptr_t distance = 0;
  return (mdbx_cursor_distance(begin, end, &distance, deepness) == MDBX_SUCCESS) ? distance : INTPTR_MIN;
}

int cursor_distribute(const MDBX_cursor *begin, const MDBX_cursor *end, MDBX_cursor **array, size_t array_size,
                      int level) {
  const intptr_t outer_diff_level = tree_diff_level(begin, end);
  if (unlikely(outer_diff_level < MDBX_RESULT_TRUE))
    return ENODATA;

  const intptr_t inner_diff_level = (outer_diff_level > MDBX_RESULT_TRUE || !inner_pointed(begin))
                                        ? MDBX_RESULT_TRUE
                                        : tree_diff_level(&begin->subcur->cursor, &end->subcur->cursor);
  if (outer_diff_level == MDBX_RESULT_TRUE && unlikely(inner_diff_level < MDBX_RESULT_TRUE))
    return ENODATA;

  if (unlikely(level < outer_diff_level))
    return MDBX_RESULT_TRUE;
  if (unlikely(outer_diff_level == MDBX_RESULT_TRUE && inner_diff_level == MDBX_RESULT_TRUE)) {
    /* Курсоры begin и end совпадают */
    return MDBX_RESULT_TRUE;
  }

  MDBX_cursor *iter = (MDBX_cursor *)begin;
  for (intptr_t i = array_size; --i >= 0;) {
    if (array[i] != begin && array[i] != end) {
      iter = array[i];
      break;
    }
  }
  cursor_copy_position(begin, iter);

  const cdr_t tdr = cursor_distance(iter, end, level);
  const size_t amount = tdr.distance;
  int rc = tdr.err;
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  ASSERT(tdr.distance > 0);
  cursor_copy_position(begin, iter);
  size_t processed = 0;
  for (size_t i = 0; i < array_size; ++i) {
    MDBX_cursor *const mc = array[i];
    if (unlikely(processed >= amount)) {
      be_poor(mc);
      rc = MDBX_RESULT_TRUE;
    } else {
      const size_t left_cursors = array_size - i;
      const size_t left_amount = amount - processed;
      const size_t chunk = (left_amount + left_cursors - 1) / left_cursors;
      int err = cursor_scroll_forward(iter, chunk, level);
      processed += chunk;
      VERBOSE("cursor %zu of %zu, chunk %zu of %zu, left %zu, distance %zi, expected %zi, err %i", i, array_size, chunk,
              amount, amount - processed, processed, debug_cursor_distance(begin, iter, level), err);
      if (unlikely(err != MDBX_SUCCESS)) {
        if (err != MDBX_NOTFOUND)
          return rc = err;
        be_poor(iter);
        processed = tdr.distance;
        rc = MDBX_RESULT_TRUE;
      } else {
        intptr_t cmp = cursor_cmp(iter, end);
        if (cmp == 0 && inner_pointed(end))
          cmp = cursor_cmp(&iter->subcur->cursor, &end->subcur->cursor);
        if (cmp > 0) {
          be_poor(iter);
          processed = tdr.distance;
          rc = MDBX_RESULT_TRUE;
        }
      }
      if (mc != iter)
        cursor_copy_position(iter, mc);
    }
  }
  return rc;
}

static defer_free_item_t *dbi_close_locked(MDBX_env *env, MDBX_dbi dbi);

#if MDBX_ENABLE_DBI_SPARSE
size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn, intptr_t bmi) {
  cASSERT0(txn, bmi != 0);
  bmi &= -bmi;
  if (sizeof(txn->dbi_sparse[0]) > 4) {
    static const uint8_t debruijn_ctz64[64] = {0,  1,  2,  53, 3,  7,  54, 27, 4,  38, 41, 8,  34, 55, 48, 28,
                                               62, 5,  39, 46, 44, 42, 22, 9,  24, 35, 59, 56, 49, 18, 29, 11,
                                               63, 52, 6,  26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
                                               51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12};
    return debruijn_ctz64[(UINT64_C(0x022FDD63CC95386D) * (uint64_t)bmi) >> 58];
  } else {
    static const uint8_t debruijn_ctz32[32] = {0,  1,  28, 2,  29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4,  8,
                                               31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6,  11, 5,  10, 9};
    return debruijn_ctz32[(UINT32_C(0x077CB531) * (uint32_t)bmi) >> 27];
  }
}
#endif /* MDBX_ENABLE_DBI_SPARSE */

struct dbi_snap_result dbi_snap(const MDBX_env *env, const size_t dbi) {
  eASSERT0(env, dbi < env->n_dbi);
  struct dbi_snap_result r;
  uint32_t snap = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease);
  do {
    r.sequence = snap;
    r.flags = env->dbs_flags[dbi];
    snap = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease);
  } while (unlikely(snap != r.sequence));
  return r;
}

int dbi_gone(MDBX_txn *txn, const size_t dbi, const int rc) {
  cASSERT0(txn, txn->n_dbi > dbi && F_ISSET(txn->dbi_state[dbi], DBI_LINDO | DBI_VALID));
  for (;;) {
    unsigned state = txn->dbi_state[dbi];
    txn->dbi_state[dbi] = DBI_OLDEN | DBI_LINDO;
    if (state & (DBI_FRESH | DBI_CREAT))
      return rc;
    if (!txn->parent)
      break;
    txn = txn->parent;
  }

  txn->dbi_seqs[dbi] = 0;
  return rc;
}

__noinline int dbi_import(MDBX_txn *txn, const size_t dbi) {
  const MDBX_env *const env = txn->env;
  if (dbi >= env->n_dbi || !env->dbs_flags[dbi])
    return MDBX_BAD_DBI;

#if MDBX_ENABLE_DBI_SPARSE
  const size_t bitmap_chunk = CHAR_BIT * sizeof(txn->dbi_sparse[0]);
  const size_t bitmap_indx = dbi / bitmap_chunk;
  const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk;
  if (dbi >= txn->n_dbi) {
    for (size_t i = (txn->n_dbi + bitmap_chunk - 1) / bitmap_chunk; bitmap_indx >= i; ++i)
      txn->dbi_sparse[i] = 0;
    eASSERT0(env, (txn->dbi_sparse[bitmap_indx] & bitmap_mask) == 0);
    MDBX_txn *scan = txn;
    do {
      eASSERT0(env, scan->dbi_sparse == txn->dbi_sparse);
      eASSERT0(env, scan->n_dbi < dbi + 1);
      scan->n_dbi = (unsigned)dbi + 1;
      scan->dbi_state[dbi] = 0;
      scan = scan->parent;
    } while (scan /* && scan->dbi_sparse == txn->dbi_sparse */);
    txn->dbi_sparse[bitmap_indx] |= bitmap_mask;
    goto lindo;
  }
  if ((txn->dbi_sparse[bitmap_indx] & bitmap_mask) == 0) {
    MDBX_txn *scan = txn;
    do {
      eASSERT0(env, scan->dbi_sparse == txn->dbi_sparse);
      eASSERT0(env, scan->n_dbi == txn->n_dbi);
      scan->dbi_state[dbi] = 0;
      scan = scan->parent;
    } while (scan /* && scan->dbi_sparse == txn->dbi_sparse */);
    txn->dbi_sparse[bitmap_indx] |= bitmap_mask;
    goto lindo;
  }
#else
  if (dbi >= txn->n_dbi) {
    size_t i = txn->n_dbi;
    do
      txn->dbi_state[i] = 0;
    while (dbi >= ++i);
    txn->n_dbi = i;
    goto lindo;
  }
#endif /* MDBX_ENABLE_DBI_SPARSE */

  if (!txn->dbi_state[dbi]) {
  lindo:
    /* dbi-слот еще не инициализирован в транзакции, а хендл не использовался */
    txn->cursors[dbi] = nullptr;
    MDBX_txn *const parent = txn->parent;
    if (unlikely(parent)) {
      /* вложенная пишущая транзакция */
      int rc = dbi_check(parent, dbi);
      /* копируем состояние dbi-хендла очищая new-флаги. */
      eASSERT0(env, txn->dbi_seqs == parent->dbi_seqs);
      txn->dbi_state[dbi] = parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY);
      if (likely(rc == MDBX_SUCCESS)) {
        txn->dbs[dbi] = parent->dbs[dbi];
        rc = txn_shadow_cursors(parent, dbi);
      }
      return rc;
    }
    txn->dbi_seqs[dbi] = 0;
    txn->dbi_state[dbi] = DBI_LINDO;
  } else {
    eASSERT0(env, txn->dbi_seqs[dbi] != env->dbi_seqs[dbi].weak);
    if (unlikely(txn->cursors[dbi])) {
      /* хендл уже использовался в транзакции и остались висячие курсоры */
      txn->dbi_seqs[dbi] = env->dbi_seqs[dbi].weak;
      txn->dbi_state[dbi] = DBI_OLDEN | DBI_LINDO;
      return MDBX_DANGLING_DBI;
    }
    if (unlikely(txn->dbi_state[dbi] & (DBI_OLDEN | DBI_VALID))) {
      /* хендл уже использовался в транзакции, но был закрыт или переоткрыт,
       * висячих курсоров нет */
      txn->dbi_seqs[dbi] = env->dbi_seqs[dbi].weak;
      txn->dbi_state[dbi] = DBI_OLDEN | DBI_LINDO;
      return MDBX_BAD_DBI;
    }
  }

  /* хендл не использовался в транзакции, либо явно пере-отрывается при
   * отсутствии висячих курсоров */
  eASSERT0(env, (txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO && !txn->cursors[dbi]);

  /* читаем актуальные флаги и sequence */
  struct dbi_snap_result snap = dbi_snap(env, dbi);
  txn->dbi_seqs[dbi] = snap.sequence;
  if (snap.flags & DB_VALID) {
    txn->dbs[dbi].flags = snap.flags & DB_PERSISTENT_FLAGS;
    txn->dbi_state[dbi] = (dbi >= CORE_DBS) ? DBI_LINDO | DBI_VALID | DBI_STALE : DBI_LINDO | DBI_VALID;
    return MDBX_SUCCESS;
  }

  return MDBX_BAD_DBI;
}

int dbi_defer_release(MDBX_env *const env, defer_free_item_t *const chain) {
  size_t length = 0;
  defer_free_item_t *obsolete_chain = nullptr;
#if MDBX_ENABLE_DBI_LOCKFREE
  const uint64_t now = osal_monotime();
  defer_free_item_t **scan = &env->defer_free;
  if (env->defer_free) {
    const uint64_t threshold_1second = osal_16dot16_to_monotime(1 * 65536);
    do {
      defer_free_item_t *item = *scan;
      if (now - item->timestamp < threshold_1second) {
        scan = &item->next;
        length += 1;
      } else {
        *scan = item->next;
        item->next = obsolete_chain;
        obsolete_chain = item;
      }
    } while (*scan);
  }

  eASSERT0(env, *scan == nullptr);
  if (chain) {
    defer_free_item_t *item = chain;
    do {
      item->timestamp = now;
      item = item->next;
    } while (item);
    *scan = chain;
  }
#else  /* MDBX_ENABLE_DBI_LOCKFREE */
  obsolete_chain = chain;
#endif /* MDBX_ENABLE_DBI_LOCKFREE */

  ENSURE_OBJ(env, osal_fastmutex_release(&env->dbi_lock) == MDBX_SUCCESS);
  if (length > 42)
    osal_yield();
  while (obsolete_chain) {
    defer_free_item_t *item = obsolete_chain;
    obsolete_chain = obsolete_chain->next;
    osal_free(item);
  }
  return chain ? MDBX_SUCCESS : MDBX_BAD_DBI;
}

/* Export or close DBI handles opened in this txn. */
int dbi_update(MDBX_txn *txn, bool keep) {
  MDBX_env *const env = txn->env;
  cASSERT0(txn, (!txn->parent && txn == env->basal_txn) || !keep);
  bool locked = false;
  defer_free_item_t *defer_chain = nullptr;
  TXN_FOREACH_DBI_USER(txn, dbi) {
    if (likely((txn->dbi_state[dbi] & DBI_CREAT) == 0))
      continue;
    if (!locked) {
      int err = osal_fastmutex_acquire(&env->dbi_lock);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      locked = true;
      if (dbi >= env->n_dbi)
        /* хендл был закрыт из другого потока пока захватывали блокировку */
        continue;
    }
    cASSERT0(txn, dbi < env->n_dbi);
    if (keep) {
      env->dbs_flags[dbi] = txn->dbs[dbi].flags | DB_VALID;
    } else {
      uint32_t seq = dbi_seq_next(env, dbi);
      defer_free_item_t *item = env->kvs[dbi].name.iov_base;
      if (item) {
        env->dbs_flags[dbi] = 0;
        env->kvs[dbi].name.iov_len = 0;
        env->kvs[dbi].name.iov_base = nullptr;
        atomic_store32(&env->dbi_seqs[dbi], seq, mo_AcquireRelease);
        osal_flush_incoherent_cpu_writeback();
        item->next = defer_chain;
        defer_chain = item;
      } else {
        eASSERT0(env, env->kvs[dbi].name.iov_len == 0);
        eASSERT0(env, env->dbs_flags[dbi] == 0);
      }
    }
  }

  if (locked) {
    size_t i = env->n_dbi;
    eASSERT0(env, env->n_dbi >= CORE_DBS);
    while ((env->dbs_flags[i - 1] & DB_VALID) == 0) {
      --i;
      eASSERT0(env, i >= CORE_DBS);
      eASSERT0(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base);
    }
    env->n_dbi = (unsigned)i;
    dbi_defer_release(env, defer_chain);
  }
  return MDBX_SUCCESS;
}

int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func keycmp, MDBX_cmp_func datacmp) {
  const MDBX_env *const env = txn->env;
  eASSERT0(env, dbi < txn->n_dbi && dbi < env->n_dbi);
  eASSERT0(env, dbi_state(txn, dbi) & DBI_LINDO);
  eASSERT0(env, env->dbs_flags[dbi] != DB_POISON);
  if ((env->dbs_flags[dbi] & DB_VALID) == 0) {
    eASSERT0(env, !env->kvs[dbi].clc.k.cmp && !env->kvs[dbi].clc.v.cmp && !env->kvs[dbi].name.iov_len &&
                      !env->kvs[dbi].name.iov_base && !env->kvs[dbi].clc.k.lmax && !env->kvs[dbi].clc.k.lmin &&
                      !env->kvs[dbi].clc.v.lmax && !env->kvs[dbi].clc.v.lmin);
  } else {
    eASSERT0(env, !(txn->dbi_state[dbi] & DBI_VALID) || (txn->dbs[dbi].flags | DB_VALID) == env->dbs_flags[dbi]);
    eASSERT0(env, env->kvs[dbi].name.iov_base || dbi < CORE_DBS);
  }

  /* Если dbi уже использовался, то корректными считаем четыре варианта:
   * 1) user_flags равны MDBX_DB_ACCEDE
   *   = предполагаем что пользователь открывает существующую table,
   *     при этом код проверки не позволит установить другие компараторы.
   * 2) user_flags нулевые, а оба компаратора пустые/нулевые или равны текущим
   *   = предполагаем что пользователь открывает существующую table
   *     старым способом с нулевыми с флагами по-умолчанию.
   * 3) user_flags совпадают, а компараторы не заданы или те же
   *    = предполагаем что пользователь открывает table указывая все параметры;
   * 4) user_flags отличаются, но table пустая и задан флаг MDBX_CREATE
   *    = предполагаем что пользователь пересоздает table;
   */
  if ((user_flags ^ env->dbs_flags[dbi]) & DB_PERSISTENT_FLAGS) {
    /* flags are differs, check other conditions */
    if (((!keycmp || keycmp == env->kvs[dbi].clc.k.cmp) && (!datacmp || datacmp == env->kvs[dbi].clc.v.cmp)) &&
        (user_flags & MDBX_DB_ACCEDE) != 0) {
      user_flags = env->dbs_flags[dbi] & DB_PERSISTENT_FLAGS;
    } else if ((user_flags & MDBX_CREATE) == 0)
      return /* FIXME: return extended info */ MDBX_INCOMPATIBLE;
    else {
      if (txn->dbi_state[dbi] & DBI_STALE) {
        eASSERT0(env, env->dbs_flags[dbi] & DB_VALID);
        int err = tbl_refresh(txn, dbi);
        if (unlikely(err != MDBX_NOTFOUND))
          return err;
      }
      eASSERT0(env, ((env->dbs_flags[dbi] ^ txn->dbs[dbi].flags) & DB_PERSISTENT_FLAGS) == 0);
      eASSERT0(env, (txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID));
      if (unlikely(txn->dbs[dbi].leaf_pages))
        return /* FIXME: return extended info */ MDBX_INCOMPATIBLE;

      /* Пересоздаём table если там пусто */
      if (unlikely(txn->cursors[dbi]))
        return MDBX_DANGLING_DBI;
      env->dbs_flags[dbi] = DB_POISON;
      atomic_store32(&env->dbi_seqs[dbi], dbi_seq_next(env, dbi), mo_AcquireRelease);

      const uint32_t seq = dbi_seq_next(env, dbi);
      const uint16_t db_flags = user_flags & DB_PERSISTENT_FLAGS;
      eASSERT0(env, txn->dbs[dbi].height == 0 && txn->dbs[dbi].items == 0 && txn->dbs[dbi].root == P_INVALID);
      env->kvs[dbi].clc.k.cmp = keycmp;
      env->kvs[dbi].clc.v.cmp = datacmp;
      txn->dbs[dbi].flags = db_flags;
      txn->dbs[dbi].dupfix_size = 0;
      if (unlikely(tbl_setup(env, &env->kvs[dbi], &txn->dbs[dbi]))) {
        txn->dbi_state[dbi] = DBI_LINDO;
        txn->flags |= MDBX_TXN_ERROR;
        return MDBX_PROBLEM;
      }

      env->dbs_flags[dbi] = db_flags | DB_VALID;
      atomic_store32(&env->dbi_seqs[dbi], seq, mo_AcquireRelease);
      txn->dbi_seqs[dbi] = seq;
      txn->dbi_state[dbi] = DBI_LINDO | DBI_VALID | DBI_CREAT | DBI_DIRTY;
      txn->flags |= MDBX_TXN_DIRTY;
      return MDBX_SUCCESS;
    }
  }

  if (!keycmp)
    keycmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.k.cmp : builtin_keycmp(user_flags);
  if (env->kvs[dbi].clc.k.cmp != keycmp) {
    if (env->dbs_flags[dbi] & DB_VALID)
      return MDBX_EINVAL;
    env->kvs[dbi].clc.k.cmp = keycmp;
    clc_reset_methods(&env->kvs[dbi].clc.k);
  }

  if (!datacmp)
    datacmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.v.cmp : builtin_datacmp(user_flags);
  if (env->kvs[dbi].clc.v.cmp != datacmp) {
    if (env->dbs_flags[dbi] & DB_VALID)
      return MDBX_EINVAL;
    env->kvs[dbi].clc.v.cmp = datacmp;
    clc_reset_methods(&env->kvs[dbi].clc.v);
  }

  return MDBX_SUCCESS;
}

static inline size_t dbi_namelen(const MDBX_val name) {
  return (name.iov_len > sizeof(defer_free_item_t)) ? name.iov_len : sizeof(defer_free_item_t);
}

static int dbi_open_locked(MDBX_txn *txn, cursor_couple_t *maindb_cx, unsigned user_flags, MDBX_cmp_func keycmp,
                           MDBX_cmp_func datacmp, MDBX_val name, const size_t fastpath_slot,
                           defer_free_item_t **defer_chain) {
  MDBX_env *const env = txn->env;
  *defer_chain = nullptr;
  int rc;

  /* Cannot mix named table(s) with DUPSORT flags */
  cASSERT0(txn, (txn->dbi_state[MAIN_DBI] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID));
  if (unlikely(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT)) {
    if (unlikely((user_flags & MDBX_CREATE) == 0))
      return MDBX_NOTFOUND;
    if (unlikely(txn->dbs[MAIN_DBI].leaf_pages))
      /* В MainDB есть записи, либо она уже использовалась. */
      return MDBX_INCOMPATIBLE;

    /* Пересоздаём MainDB когда там пусто. */
    cASSERT0(txn,
             txn->dbs[MAIN_DBI].height == 0 && txn->dbs[MAIN_DBI].items == 0 && txn->dbs[MAIN_DBI].root == P_INVALID);
    if (unlikely(txn->cursors[MAIN_DBI]))
      return MDBX_DANGLING_DBI;
    env->dbs_flags[MAIN_DBI] = DB_POISON;
    atomic_store32(&env->dbi_seqs[MAIN_DBI], dbi_seq_next(env, MAIN_DBI), mo_AcquireRelease);

    const uint32_t seq = dbi_seq_next(env, MAIN_DBI);
    const uint16_t main_flags = txn->dbs[MAIN_DBI].flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY);
    env->kvs[MAIN_DBI].clc.k.cmp = builtin_keycmp(main_flags);
    env->kvs[MAIN_DBI].clc.v.cmp = builtin_datacmp(main_flags);
    txn->dbs[MAIN_DBI].flags = main_flags;
    txn->dbs[MAIN_DBI].dupfix_size = 0;
    rc = tbl_setup(env, &env->kvs[MAIN_DBI], &txn->dbs[MAIN_DBI]);
    if (unlikely(rc != MDBX_SUCCESS)) {
      txn->dbi_state[MAIN_DBI] = DBI_LINDO;
      txn->flags |= MDBX_TXN_ERROR;
      env->flags |= ENV_FATAL_ERROR;
      return rc;
    }
    env->dbs_flags[MAIN_DBI] = main_flags | DB_VALID;
    txn->dbi_seqs[MAIN_DBI] = atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease);
    txn->dbi_state[MAIN_DBI] |= DBI_DIRTY;
    txn->flags |= MDBX_TXN_DIRTY;
  }

  cASSERT0(txn, env->kvs[MAIN_DBI].clc.k.cmp);

  /* Is the DB already open? */
  defer_free_item_t *clone = nullptr;
  size_t slot = env->n_dbi;
  for (size_t scan = CORE_DBS; scan < env->n_dbi; ++scan) {
    if ((env->dbs_flags[scan] & DB_VALID) == 0) {
      /* Remember this free slot */
      slot = (slot < scan) ? slot : scan;
      continue;
    }
    if (env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[scan].name) == 0) {
      slot = scan;
      rc = dbi_check(txn, slot);
      if (rc == MDBX_BAD_DBI &&
          (txn->dbi_state[slot] ==
               /* хендл использовался, стал невалидным, но теперь явно пере-открывается */ (DBI_OLDEN | DBI_LINDO) ||
           (txn->dbi_state[slot] ==
            /* хендл был инициализирован в дочерней транзакции, но она была прервана */ DBI_LINDO))) {
        eASSERT0(env, !txn->cursors[slot]);
        txn->dbi_state[slot] = DBI_LINDO;
        txn->dbi_seqs[slot] = 0;
        rc = dbi_import(txn, slot);
      }
      if (unlikely(rc != MDBX_SUCCESS))
        goto gone;

      rc = dbi_bind(txn, slot, user_flags, keycmp, datacmp);
      if (unlikely(rc != MDBX_SUCCESS))
        goto gone;

      if (unlikely((txn->dbi_state[slot] & DBI_STALE) == 0))
        goto done;

      if (fastpath_slot /* уже был выполнен поиск посредством tbl_fetch() */) {
        if (slot != fastpath_slot)
          txn->dbs[slot] = txn->dbs[fastpath_slot];
        if (user_flags & MDBX_CREATE) {
          /* значит таблица уже была открытой, но проверка её наличия в fastpath вернула MDBX_NOTFOUND */
          rc = MDBX_NOTFOUND;
        } else {
          /* значит в fastpath был найден пустой слот и проверка наличия таблицы завершилась успешно */
          ASSERT(rc == MDBX_SUCCESS);
        }
      } else {
        rc = tbl_fetch(txn, &maindb_cx->outer, slot, &name, user_flags);
      }

      if (likely(rc == MDBX_SUCCESS))
        goto done;

      if (rc == MDBX_NOTFOUND && (user_flags & MDBX_CREATE)) {
        name = env->kvs[scan].name;
        goto create;
      }

    gone:
      *defer_chain = dbi_close_locked(env, slot);
      return rc;
    }
  }

  /* Fail, if no free slot and max hit */
  if (unlikely(slot >= env->max_dbi))
    return MDBX_DBS_FULL;

  if (env->n_dbi == slot)
    eASSERT0(env, !env->dbs_flags[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base);

  env->dbs_flags[slot] = DB_POISON;
  atomic_store32(&env->dbi_seqs[slot], dbi_seq_next(env, slot), mo_AcquireRelease);
  memset(&env->kvs[slot], 0, sizeof(env->kvs[slot]));
  if (env->n_dbi == slot)
    env->n_dbi = (unsigned)slot + 1;
  eASSERT0(env, slot < env->n_dbi);

  rc = dbi_check(txn, slot);
  eASSERT0(env, rc == MDBX_BAD_DBI);
  if (unlikely(rc != MDBX_BAD_DBI))
    return MDBX_PROBLEM;

  /* Find the DB info */
#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
  void *preserve_userctx = maindb_cx->userctx;
#endif /* MEMCHECK || ASAN */
  rc = tbl_fetch(txn, &maindb_cx->outer, slot, &name, user_flags);
#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
  maindb_cx->userctx = preserve_userctx;
#endif /* MEMCHECK || ASAN */
  if (unlikely(rc != MDBX_SUCCESS)) {
    if (rc != MDBX_NOTFOUND || !(user_flags & MDBX_CREATE))
      return rc;
  }

  /* Done here so we cannot fail after creating a new DB */
  clone = osal_malloc(dbi_namelen(name));
  if (unlikely(!clone))
    return MDBX_ENOMEM;
  memcpy(clone, name.iov_base, name.iov_len);
  name.iov_base = clone;

create:
  cASSERT0(txn, rc == MDBX_SUCCESS || rc == MDBX_NOTFOUND);
  uint8_t dbi_state = DBI_LINDO | DBI_VALID | DBI_FRESH;
  if (unlikely(rc != MDBX_SUCCESS)) {
    rc = tbl_create(txn, &maindb_cx->outer, slot, &name, user_flags);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;
    dbi_state |= DBI_DIRTY | DBI_CREAT;
  }

  /* Got info, register DBI in this txn */
  const uint32_t seq = dbi_seq_next(env, slot);
  eASSERT0(env, !txn->cursors[slot]);
  if (clone) {
    eASSERT0(env, env->dbs_flags[slot] == DB_POISON && (txn->dbi_state[slot] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO);
    txn->dbi_state[slot] = dbi_state;
    env->dbs_flags[slot] = txn->dbs[slot].flags;
    rc = dbi_bind(txn, slot, user_flags, keycmp, datacmp);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;

    env->kvs[slot].name = name;
    env->dbs_flags[slot] = txn->dbs[slot].flags | DB_VALID;
    txn->dbi_seqs[slot] = atomic_store32(&env->dbi_seqs[slot], seq, mo_AcquireRelease);
  } else {
    eASSERT0(env, env->dbs_flags[slot] == (DB_VALID | (user_flags & DB_PERSISTENT_FLAGS)) &&
                      env->dbs_flags[slot] == (DB_VALID | txn->dbs[slot].flags) &&
                      txn->dbi_state[slot] == (DBI_LINDO | DBI_VALID | DBI_STALE));
  }

done:
  *(MDBX_dbi *)maindb_cx->userctx = (MDBX_dbi)slot;
  tASSERT0(txn, slot < txn->n_dbi && (env->dbs_flags[slot] & DB_VALID) != 0);
  eASSERT0(env, dbi_check(txn, slot) == MDBX_SUCCESS);
  return MDBX_SUCCESS;

bailout:
  txn->dbi_state[slot] &= DBI_LINDO | DBI_OLDEN;
  env->dbs_flags[slot] = 0;
  if (clone) {
    eASSERT0(env, !txn->cursors[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base);
    osal_free(clone);
    if (slot + 1 == env->n_dbi)
      txn->n_dbi = env->n_dbi = (unsigned)slot;
  } else {
    eASSERT0(env, name.iov_base == env->kvs[slot].name.iov_base);
  }
  return rc;
}

int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func keycmp,
             MDBX_cmp_func datacmp) {
  if (unlikely(!dbi))
    return MDBX_EINVAL;
  *dbi = 0;

  if (user_flags != MDBX_ACCEDE && unlikely(!check_table_flags(user_flags & ~MDBX_CREATE)))
    return MDBX_EINVAL;

  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  if ((user_flags & MDBX_CREATE) && unlikely(txn->flags & txn_ro_both))
    return MDBX_EACCESS;

  /* main table? */
  if (unlikely(name == MDBX_CHK_MAIN || name->iov_base == MDBX_CHK_MAIN)) {
    rc = dbi_bind(txn, MAIN_DBI, user_flags, keycmp, datacmp);
    if (likely(rc == MDBX_SUCCESS))
      *dbi = MAIN_DBI;
    return rc;
  }
  if (unlikely(name == MDBX_CHK_GC || name->iov_base == MDBX_CHK_GC)) {
    rc = dbi_bind(txn, FREE_DBI, user_flags, keycmp, datacmp);
    if (likely(rc == MDBX_SUCCESS))
      *dbi = FREE_DBI;
    return rc;
  }
  if (unlikely(name == MDBX_CHK_META || name->iov_base == MDBX_CHK_META))
    return MDBX_EINVAL;
  if (unlikely(name->iov_len > txn->env->leaf_nodemax - NODESIZE - sizeof(tree_t)))
    return MDBX_EINVAL;

  cursor_couple_t cx;
  size_t fastpath_slot = 0;
#if MDBX_ENABLE_DBI_LOCKFREE
  /* Is the DB already open? */
  const MDBX_env *const env = txn->env;
  size_t first_free_slot = env->n_dbi;
  for (size_t slot = CORE_DBS; slot < env->n_dbi; ++slot) {
    if ((env->dbs_flags[slot] & DB_VALID) == 0) {
      first_free_slot = (first_free_slot < slot) ? first_free_slot : slot;
      continue;
    }

    struct dbi_snap_result snap = dbi_snap(env, slot);
    const MDBX_val snap_name = env->kvs[slot].name;
    const uint32_t main_seq = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease);
    const MDBX_cmp_func snap_cmp = env->kvs[MAIN_DBI].clc.k.cmp;
    if (unlikely(!(snap.flags & DB_VALID) || !snap_name.iov_base || !snap_name.iov_len || !snap_cmp))
      /* похоже на столкновение с параллельно работающим обновлением */
      goto slowpath_locking;

    const bool name_match = snap_cmp(&snap_name, name) == 0;
    if (unlikely(snap.sequence != atomic_load32(&env->dbi_seqs[slot], mo_AcquireRelease) ||
                 main_seq != atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) ||
                 snap.flags != env->dbs_flags[slot] || snap_name.iov_base != env->kvs[slot].name.iov_base ||
                 snap_name.iov_len != env->kvs[slot].name.iov_len))
      /* похоже на столкновение с параллельно работающим обновлением */
      goto slowpath_locking;

    if (!name_match)
      continue;

    osal_flush_incoherent_cpu_writeback();
    if (user_flags != MDBX_ACCEDE &&
        (((user_flags ^ snap.flags) & DB_PERSISTENT_FLAGS) || (keycmp && keycmp != env->kvs[slot].clc.k.cmp) ||
         (datacmp && datacmp != env->kvs[slot].clc.v.cmp)))
      /* есть подозрение что пользователь открывает таблицу с другими флагами/атрибутами
       * или другими компараторами, поэтому уходим в безопасный режим */
      goto slowpath_locking;

    rc = dbi_check(txn, slot);
    if (rc == MDBX_BAD_DBI &&
        (txn->dbi_state[slot] ==
             /* хендл использовался, стал невалидным, но теперь явно пере-открывается */ (DBI_OLDEN | DBI_LINDO) ||
         (txn->dbi_state[slot] ==
          /* хендл был инициализирован в дочерней транзакции, но она была прервана */ DBI_LINDO)))
      goto slowpath_locking;
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;

    if (unlikely(snap.sequence != atomic_load32(&env->dbi_seqs[slot], mo_AcquireRelease) ||
                 main_seq != atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) ||
                 snap.flags != env->dbs_flags[slot] || snap_name.iov_base != env->kvs[slot].name.iov_base ||
                 snap_name.iov_len != env->kvs[slot].name.iov_len))
      /* похоже на столкновение с параллельно работающим обновлением */
      goto slowpath_locking;

    rc = dbi_bind(txn, slot, user_flags, keycmp, datacmp);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;

    cASSERT0(txn, F_ISSET(txn->dbi_state[slot], DBI_LINDO | DBI_VALID));
    if (txn->dbi_state[slot] & DBI_STALE) {
      rc = tbl_fetch(txn, &cx.outer, fastpath_slot = slot, name, user_flags);
      if (unlikely(rc != MDBX_SUCCESS)) {
        /* таблица отсутствует, её нужно создавать (если запрос с MDBX_CREATE), либо освобождать слот. */
        goto slowpath_locking;
      }
      txn->dbi_state[slot] -= DBI_STALE;
    }
    *dbi = (MDBX_dbi)slot;
    return MDBX_SUCCESS;
  }

  /* Fail, if no free slot and max hit */
  if (unlikely(first_free_slot >= env->max_dbi))
    return MDBX_DBS_FULL;

  if (!(user_flags & MDBX_CREATE)) {
    rc = tbl_fetch(txn, &cx.outer, fastpath_slot = first_free_slot, name, user_flags);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }

slowpath_locking:
#endif /* MDBX_ENABLE_DBI_LOCKFREE */

  cx.userctx = dbi;
  rc = osal_fastmutex_acquire(&txn->env->dbi_lock);
  if (likely(rc == MDBX_SUCCESS)) {
    defer_free_item_t *defer_chain = nullptr;
    rc = dbi_open_locked(txn, &cx, user_flags, keycmp, datacmp, *name, fastpath_slot, &defer_chain);
    dbi_defer_release(txn->env, defer_chain);
  }
  return rc;
}

__cold struct dbi_rename_result dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) {
  struct dbi_rename_result pair;
  pair.defer = nullptr;
  pair.err = dbi_check(txn, dbi);
  if (unlikely(pair.err != MDBX_SUCCESS))
    return pair;

  MDBX_env *const env = txn->env;
  MDBX_val old_name = env->kvs[dbi].name;
  if (env->kvs[MAIN_DBI].clc.k.cmp(&new_name, &old_name) == 0 && MDBX_DEBUG < 1)
    return pair;

  cursor_couple_t cx;
  pair.err = cursor_init(&cx.outer, txn, MAIN_DBI);
  if (unlikely(pair.err != MDBX_SUCCESS))
    return pair;
  pair.err = cursor_seek(&cx.outer, &new_name, nullptr, MDBX_SET).err;
  if (unlikely(pair.err != MDBX_NOTFOUND)) {
    pair.err = (pair.err == MDBX_SUCCESS) ? MDBX_KEYEXIST : pair.err;
    return pair;
  }

  pair.defer = osal_malloc(dbi_namelen(new_name));
  if (unlikely(!pair.defer)) {
    pair.err = MDBX_ENOMEM;
    return pair;
  }
  new_name.iov_base = memcpy(pair.defer, new_name.iov_base, new_name.iov_len);

  cx.outer.next = txn->cursors[MAIN_DBI];
  txn->cursors[MAIN_DBI] = &cx.outer;

  MDBX_val data = {&txn->dbs[dbi], sizeof(tree_t)};
  pair.err = cursor_put_checklen(&cx.outer, &new_name, &data, N_TREE | MDBX_NOOVERWRITE);
  if (likely(pair.err == MDBX_SUCCESS)) {
    pair.err = cursor_seek(&cx.outer, &old_name, nullptr, MDBX_SET).err;
    if (likely(pair.err == MDBX_SUCCESS))
      pair.err = cursor_del(&cx.outer, N_TREE);
    if (likely(pair.err == MDBX_SUCCESS)) {
      pair.defer = env->kvs[dbi].name.iov_base;
      env->kvs[dbi].name = new_name;
    } else
      txn->flags |= MDBX_TXN_ERROR;
  }

  txn->cursors[MAIN_DBI] = cx.outer.next;
  return pair;
}

static defer_free_item_t *dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) {
  eASSERT0(env, dbi >= CORE_DBS);
  if (unlikely(dbi >= env->n_dbi))
    return nullptr;

  const uint32_t seq = dbi_seq_next(env, dbi);
  defer_free_item_t *defer_item = env->kvs[dbi].name.iov_base;
  if (likely(defer_item)) {
    env->dbs_flags[dbi] = 0;
    env->kvs[dbi].name.iov_len = 0;
    env->kvs[dbi].name.iov_base = nullptr;
    atomic_store32(&env->dbi_seqs[dbi], seq, mo_AcquireRelease);
    osal_flush_incoherent_cpu_writeback();
    defer_item->next = nullptr;

    if (env->n_dbi == dbi + 1) {
      size_t i = env->n_dbi;
      do {
        --i;
        eASSERT0(env, i >= CORE_DBS);
        eASSERT0(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base);
      } while (i > CORE_DBS && !env->kvs[i - 1].name.iov_base);
      env->n_dbi = (unsigned)i;
    }
  }

  return defer_item;
}

__cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback) {
  const MDBX_txn *dig = txn;
  do {
    cASSERT0(txn, txn->n_dbi == dig->n_dbi);
    const uint8_t state = dbi_state(dig, dbi);
    if (state & DBI_LINDO)
      switch (state & (DBI_VALID | DBI_STALE | DBI_OLDEN)) {
      case DBI_VALID:
      case DBI_OLDEN:
        return dig->dbs + dbi;
      case 0:
        return fallback;
      case DBI_VALID | DBI_STALE:
      case DBI_OLDEN | DBI_STALE:
        break;
      default:
        cASSERT0(txn, !!"unexpected dig->dbi_state[dbi]");
      }
    dig = dig->parent;
  } while (dig);
  return fallback;
}

int dbi_close_release(MDBX_env *env, MDBX_dbi dbi) { return dbi_defer_release(env, dbi_close_locked(env, dbi)); }

static uint64_t defrag_now(uint64_t now_cache) { return now_cache ? now_cache : osal_monotime(); }

uint64_t defrag_result(dfc_t *dfc, MDBX_defrag_result_t *out, uint64_t now_cache) {
  memset(out, 0, sizeof(*out));
  if (dfc->txn)
    dfc->last_allocated = dfc->txn->geo.first_unallocated;
  out->pages_shrinked = dfc->before_defrag - dfc->last_allocated;
  out->pages_moved = dfc->cycle_pages_moved;
  out->pages_scheduled = dfc->cycle_pages_scheduled;
  out->pages_retained = dfc->gc_retained_pages;
  if (dfc->stopor)
    out->pages_retained += dfc->gc_tree_pages;

  intptr_t pages_left = dfc->last_allocated - (dfc->payload_pages + out->pages_retained);
  out->pages_left = (pages_left > 0) ? pages_left : 0;
  out->pages_whole = dfc->before_defrag;

  size_t denominator = (dfc->txn ? dfc->txn->dbs[FREE_DBI].items + rkl_len(&dfc->txn->wr.gc.ready4reuse) : 0) +
                       (dfc->cycle ? (dfc->last_allocated - dfc->payload_pages) * 2 + dfc->cycle_preprogress
                                   : out->pages_whole - NUM_METAS);
  out->rough_estimation_cycle_progress_permille =
      (dfc->progress_counter < denominator) ? (unsigned)(dfc->progress_counter * UINT64_C(1000) / denominator) : 1000;

  if (MDBX_DEBUG > 0 && dfc->progress_counter > denominator && dfc->txn) {
    WARNING("progress_counter %zu > denominator %zu | gc-items %" PRIu64 ", rkl-ready4reuse %zu | "
            "last_allocated %u, "
            "payload_pages %u, cycle_pages_scheduled %u | pages_whole %zu, walk_cutoff %u",
            dfc->progress_counter, denominator, dfc->txn->dbs[FREE_DBI].items, rkl_len(&dfc->txn->wr.gc.ready4reuse),
            dfc->last_allocated, dfc->payload_pages, dfc->cycle_pages_scheduled, out->pages_whole, dfc->walk_cutoff);
  }

  out->obstructed_pgno = dfc->stumble_pgno;
  out->obstructed_span = dfc->stumble_pgno ? dfc->stumble_span : 0;
  out->obstructed_txnid = dfc->gc_obstacle.txnid;
  out->obstructor_tid = dfc->gc_obstacle.tid;
  out->obstructor_pid = dfc->gc_obstacle.pid;
  out->cycles = dfc->cycle;
  out->stopping_reasons = dfc->stopping_reasons;
  out->spent_time_dot16 =
      osal_monotime_to_16dot16_noUnderflow((now_cache = defrag_now(now_cache)) - dfc->start_timestamp);
  return now_cache;
}

static bool defrag_gc_empty(const dfc_t *dfc) { return dfc->txn->dbs[FREE_DBI].items == 0; }

static uint64_t defrag_notify(dfc_t *dfc) {
  uint64_t now_cache = 0;
  if (dfc->user_callback) {
    MDBX_defrag_result_t progress;
    now_cache = defrag_result(dfc, &progress, now_cache);
    const int feedback = dfc->user_callback(dfc->user_ctx, &progress);
    if (feedback != 0)
      dfc->stopping_reasons |= (feedback > 0) ? MDBX_defrag_discontinued : MDBX_defrag_aborted;
  }
  return now_cache;
}

static void defrag_stumble(dfc_t *dfc, const da_t *at, const char *reason_prefix, ptrdiff_t reason_value,
                           const char *reason_suffix) {
  if (dfc->stumble_pgno < at->key_or_pgno) {
    dfc->stumble_pgno = at->key_or_pgno;
    dfc->stumble_span = at->npages;
  }
  NOTICE("defragmentation cycle %u (moved %+i pages, %+i scheduled), stumbled on the page/chunk %" PRIaPGNO
         "[%u], because %s%zi%s.",
         dfc->cycle, dfc->cycle_pages_moved, dfc->cycle_pages_scheduled, at->key_or_pgno, at->npages, reason_prefix,
         reason_value, reason_suffix);
  defrag_milestone(dfc);
}

void defrag_milestone(dfc_t *dfc) { defrag_notify(dfc); }

static void defrag_progress(dfc_t *dfc, size_t progress_increment) {
  uint64_t now_cache = (progress_increment && ((dfc->progress_counter += progress_increment) & dfc->notify_watchmask))
                           ? 0
                           : defrag_notify(dfc);

  if (dfc->stopping_reasons < MDBX_defrag_discontinued && dfc->txn &&
      dfc->defrag_atleast >= dfc->txn->geo.first_unallocated) {
    if (dfc->defrag_enough >= dfc->txn->geo.first_unallocated) {
      if (!dfc->wallclock_atleast || (now_cache = defrag_now(now_cache)) >= dfc->wallclock_atleast)
        dfc->stopping_reasons |= MDBX_defrag_enough_threshold;
    } else if (dfc->wallclock_detent && ++dfc->wallclock_trottle % 64 == 0 &&
               (now_cache = defrag_now(now_cache)) >= dfc->wallclock_detent)
      dfc->stopping_reasons |= MDBX_defrag_time_limit;
  }
}

static bool defrag_should_continue(dfc_t *dfc, size_t progress_increment) {
  defrag_progress(dfc, progress_increment);
  return !defrag_discontinued(dfc);
}

uint64_t defrag_score(dfc_t *dfc, size_t allocated_pages) {
  MDBX_txn *const txn = dfc->txn;
  const const_pnl_t pnl = txn->wr.repnl;
  const tree_t *gc = &dfc->txn->dbs[FREE_DBI];

  /* Считаем рейтинг "полезности" состояния GC, чтобы избежать бессмысленного обновления/коммита.
   * Состояние однозначно лучше, если уменьшилось количество распределенных страниц или в GC стало больше
   * последовательностей. Используем только reclaiming-список, в который должно быть загружено всё состояние GC. */
  dfc->gc_tree_pages = gc->branch_pages + gc->leaf_pages + gc->large_pages;
  uint64_t score = MAX_PAGENO - allocated_pages;
  score <<= 32;
  score -= (gc->items + gc->height + dfc->gc_tree_pages) << 5;
  for (size_t span, i = 1; i <= pnl_size(pnl); i += span) {
    span = pnl_scan_span(pnl, i);
    span = (span < UINT16_MAX) ? span : UINT16_MAX;
    score += span * span - 2;
  }
  return score;
}

static int defrag_load_gc(dfc_t *dfc) {
  MDBX_val gc_key, gc_data;
  MDBX_txn *const txn = dfc->txn;

  txn_gc_detent(txn);
  dfc->stopor = 0;
  dfc->gc_retained_pages = 0;
  ASSERT((dfc->stopping_reasons & MDBX_defrag_laggard_reader) == 0);

  MDBX_cursor *const gc = gc_cursor_init(txn);
  int rc = outer_first(gc, &gc_key, &gc_data);
  while (rc == MDBX_SUCCESS) {
    if (unlikely(gc_check_keylen(gc_key.iov_len))) {
      ERROR("%s/%d: %s", "corrupted GC-record", rc = MDBX_CORRUPTED, gc_check_keylen(gc_key.iov_len));
      break;
    }
    const glr_t glr = gc_row_pnl(txn, gc_data);
    if (unlikely(glr.err != MDBX_SUCCESS)) {
      ERROR("%s/%d: %s", "corrupted GC-record", rc = glr.err, glr.reason);
      break;
    }
    const txnid_t id = unaligned_peek_u64(4, gc_key.iov_base);
    if (id > txn->env->gc.detent) {
      if (!dfc->stopor) {
        if (id <= txn_basis_snapshot(txn)) {
          if (meta_prefer_steady(txn->env, &txn->wr.troika).txnid == txn->env->gc.detent) {
            /* make steady checkpoint. */
#if MDBX_ENABLE_PROFGC
            txn->env->lck->pgops.gc_prof.flushes += 1;
#endif /* MDBX_ENABLE_PROFGC */
            const meta_ptr_t recent = meta_recent(txn->env, &txn->wr.troika);
            ASSERT(recent.txnid > txn->env->gc.detent);
            meta_t meta = *recent.ptr_c;
            rc = dxb_sync_locked(txn->env, txn->env->flags & (MDBX_WRITEMAP | MDBX_NOMETASYNC), &meta, &txn->wr.troika);
            if (unlikely(rc != MDBX_SUCCESS))
              break;
          }
          if (txn_gc_detent(txn))
            continue;
          if (mvcc_kick_laggards(txn, txn->env->gc.detent, &dfc->gc_obstacle) && txn_gc_detent(txn))
            continue;
        }
        dfc->stopor = id;
        dfc->stopping_reasons |= MDBX_defrag_laggard_reader;
      }
      dfc->gc_retained_pages += pnl_size(glr.pnl);
    } else if (!gc_is_reclaimed(txn, id)) {
      rc = rkl_push(&txn->wr.gc.reclaimed, id);
      if (unlikely(rc != MDBX_SUCCESS))
        break;
      rc = pnl_append_pnl(&txn->wr.repnl, glr.pnl);
      if (unlikely(rc != MDBX_SUCCESS))
        break;
    }

    if (!defrag_should_continue(dfc, pnl_size(glr.pnl))) {
      rc = MDBX_RESULT_TRUE;
      break;
    }

    rc = outer_next(gc, &gc_key, &gc_data, MDBX_NEXT);
  }

  if (rc == MDBX_NOTFOUND)
    rc = MDBX_SUCCESS;

  if (likely(!MDBX_IS_ERROR(rc))) {
    pnl_sort_nochk(txn->wr.repnl);
    if (unlikely(!pnl_check(txn->wr.repnl, txn->geo.first_unallocated)))
      return MDBX_PROBLEM;
  }

  return rc;
}

static int defrag_clear_reclaimed(dfc_t *dfc) {
  int rc = MDBX_SUCCESS;
  MDBX_txn *const txn = dfc->txn;
  if (!rkl_empty(&txn->wr.gc.reclaimed)) {
    MDBX_cursor *const gc = gc_cursor_init(txn);
    if (txn->dbs[FREE_DBI].items == rkl_len(&txn->wr.gc.reclaimed)) {
      rc = tbl_purge(gc);
      if (likely(rc == MDBX_SUCCESS)) {
        defrag_progress(dfc, /* TODO? */ rkl_len(&txn->wr.gc.reclaimed));
        rc = rkl_merge(&txn->wr.gc.reclaimed, &txn->wr.gc.ready4reuse, false);
        rkl_clear(&txn->wr.gc.reclaimed);
      }
    } else if (gc_may_clean_reclaimed(txn)) {
      gc->next = txn->cursors[FREE_DBI];
      txn->cursors[FREE_DBI] = gc;
      txn_refund(txn);
      do {
        MDBX_val gc_key;
        rc = outer_first(gc, &gc_key, nullptr);
        if (unlikely(rc != MDBX_SUCCESS))
          break;
        if (unlikely(gc_check_keylen(gc_key.iov_len))) {
          ERROR("%s/%d: %s", "corrupted GC-record", rc = MDBX_CORRUPTED, gc_check_keylen(gc_key.iov_len));
          break;
        }

        const txnid_t gc_first = unaligned_peek_u64(4, gc_key.iov_base);
        const txnid_t reclaimed_lower = rkl_pop(&txn->wr.gc.reclaimed, false);
        if (gc_first != reclaimed_lower) {
          ERROR("unexpected first-gc %" PRIaTXN " != reclaimed-lower %" PRIaTXN, gc_first, reclaimed_lower);
          rc = MDBX_PROBLEM;
          break;
        }

        if (txn->wr.loose_count > 0) {
          rc = gc_merge_loose(txn);
          if (unlikely(rc != MDBX_SUCCESS))
            break;
          txn_refund(txn);
        }
        rc = cursor_del(gc, 0);
        if (unlikely(rc != MDBX_SUCCESS))
          break;
        rc = rkl_push(&txn->wr.gc.ready4reuse, gc_first);
        if (unlikely(rc != MDBX_SUCCESS))
          break;
        txn_refund(txn);

        defrag_progress(dfc, /* TODO? */ 1);
        if (defrag_aborted(dfc)) {
          rc = MDBX_RESULT_TRUE;
          break;
        }
      } while (!rkl_empty(&txn->wr.gc.reclaimed) && gc_may_clean_reclaimed(txn));
      txn->cursors[FREE_DBI] = gc->next;
    }
  }

  if (likely(rc == MDBX_SUCCESS)) {
    if (txn->wr.loose_count > 0)
      rc = gc_merge_loose(txn);
    if (likely(rc == MDBX_SUCCESS))
      txn_refund(txn);
  }

  return rc;
}

static int defrag_puch_arc(dfc_t *const dfc, pgno_t parent, pgno_t page, unsigned npages, bool is_gc) {
  ASSERT(parent < MAX_PAGENO && page < MAX_PAGENO && (parent == 0 || parent >= NUM_METAS) && page >= NUM_METAS);
  dfc->payload_items += !is_gc;
  da_t *arc = dml_append(&dfc->arcs, page);
  if (likely(arc)) {
    arc->parent = parent;
    arc->npages = npages;
    arc->gc = !!is_gc;
    return MDBX_SUCCESS;
  } else
    return MDBX_ENOMEM;
}

static int defrag_walker(const size_t pgno, unsigned npages, void *const ctx, const unsigned deep,
                         const walk_tbl_t *table, const size_t page_size, const page_type_t page_type,
                         const txnid_t page_txnid, MDBX_error_t err, const size_t nentries, const size_t payload_bytes,
                         const size_t header_bytes, const size_t unused_bytes, const size_t parent_pgno) {
  (void)page_size;
  (void)page_txnid;
  (void)nentries;
  (void)payload_bytes;
  (void)header_bytes;
  (void)unused_bytes;
  (void)parent_pgno;

  dfc_t *const dfc = ctx;
  if (unlikely(err != MDBX_SUCCESS))
    return err;
  if (unlikely((page_type & (P_BRANCH | P_LEAF | P_LARGE | P_DUPFIX | P_SUBP)) >= P_SUBP))
    return MDBX_SUCCESS;
  if (unlikely(deep >= ARRAY_LENGTH(dfc->walk_stack)))
    return MDBX_CORRUPTED;

/* используем старший бит для пометки уже отложенных страниц */
#define DEFRAG_TRACK_FLAG UINT32_C(0x80000000)
  STATIC_ASSERT(DEFRAG_TRACK_FLAG > MAX_PAGENO);

  ASSERT(deep == 0 || (dfc->walk_stack[deep - 1] & ~DEFRAG_TRACK_FLAG) == parent_pgno);
  dfc->walk_stack[deep] = pgno;

  if (pgno >= dfc->walk_cutoff) {
    if (npages > 1 && pgno >= dfc->defrag_enough) {
      dfc->largepage_count += 1;
      dfc->largepage_amountleft += npages;
      if (npages > dfc->largepage_max)
        dfc->largepage_max = npages;
    }
    for (intptr_t i = deep; i >= 0 && dfc->walk_stack[i] < DEFRAG_TRACK_FLAG; --i) {
      err = defrag_puch_arc(dfc, i ? dfc->walk_stack[i - 1] & ~DEFRAG_TRACK_FLAG : 0, dfc->walk_stack[i],
                            (i == (intptr_t)deep) ? npages : 1, table->internal == &dfc->txn->dbs[FREE_DBI]);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      dfc->walk_stack[i] += DEFRAG_TRACK_FLAG;
    }
  }

  return defrag_should_continue(dfc, npages) ? MDBX_SUCCESS : MDBX_RESULT_TRUE;
}

__cold static int defrag_gc_lookup_page(dfc_t *dfc, pgno_t pgno, txnid_t *id) {
  *id = INVALID_TXNID;

  MDBX_val gc_key, gc_data;
  MDBX_txn *const txn = dfc->txn;
  MDBX_cursor *const gc = gc_cursor_init(txn);
  int rc = outer_first(gc, &gc_key, &gc_data);
  while (rc == MDBX_SUCCESS) {
    if (unlikely(gc_check_keylen(gc_key.iov_len))) {
      ERROR("%s/%d: %s", "corrupted GC-record", rc = MDBX_CORRUPTED, gc_check_keylen(gc_key.iov_len));
      break;
    }
    const glr_t glr = gc_row_pnl(txn, gc_data);
    if (unlikely(glr.err != MDBX_SUCCESS)) {
      ERROR("%s/%d: %s", "corrupted GC-record", rc = glr.err, glr.reason);
      break;
    }

    if (pnl_contains(glr.pnl, pgno)) {
      *id = unaligned_peek_u64(4, gc_key.iov_base);
      return MDBX_SUCCESS;
    }
    rc = outer_next(gc, &gc_key, &gc_data, MDBX_NEXT);
  }

  return rc;
}

MDBX_MAYBE_UNUSED static pgr_t defrag_get_page(dfc_t *dfc, pgno_t pgno) {
  MDBX_txn *const txn = dfc->txn;
  ASSERT(pgno >= NUM_METAS && pgno < txn->geo.first_unallocated);
  pgr_t pgr = page_get_unchecked(txn, pgno, txn_basis_snapshot(txn));
  if (likely(pgr.err == MDBX_SUCCESS) && unlikely(pgr.page->flags & ~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE)))
    pgr.err = bad_page(pgr.page, "unexpected page flags 0x%x", pgr.page->flags);
  return pgr;
}

static int defrag_fixup_ref(dfc_t *dfc, void *pgno_ptr) {
  const pgno_t pgno = peek_pgno(pgno_ptr);
  if (pgno < NUM_METAS || pgno == P_INVALID)
    return MDBX_RESULT_FALSE;

  da_t *arc = dml_search_exact(dfc->arcs, pgno);
  if (arc) {
    if (arc->mapped) {
      poke_pgno(pgno_ptr, arc->mapped);
      return MDBX_RESULT_TRUE;
    }
    return MDBX_RESULT_FALSE;
  }
  if (pgno < dfc->walk_cutoff)
    return MDBX_RESULT_FALSE;
  ERROR("unexpected page %" PRIaPGNO " not found in the tracking map", pgno);
  return MDBX_PROBLEM;
}

static int defrag_fixup_page(dfc_t *dfc, page_t *dst, pgno_t pgno) {
  switch (dst->flags) {
  default:
    return bad_page(dst, "unexpected page flags 0x%x", dst->flags);
  case P_BRANCH:
    /* корректируем ссылки на дочерние страницы */
    for (size_t i = 0; i < page_numkeys(dst); ++i) {
      node_t *node = page_node(dst, i);
      int err;
      switch (node->flags) {
      default:
        return bad_page(dst, "unexpected node[%zu] flags 0x%x", i, node->flags);
      case 0 /* usual branch node */:
        err = defrag_fixup_ref(dfc, ptr_disp(node, offsetof(node_t, child_pgno)));
        if (unlikely(MDBX_IS_ERROR(err)))
          return err;
      }
    }
    break;
  case P_LEAF:
    /* корректируем ссылки на дочерние страницы */
    for (size_t i = 0; i < page_numkeys(dst); ++i) {
      node_t *node = page_node(dst, i);
      int err;
      switch (node->flags) {
      default:
        return bad_page(dst, "unexpected node[%zu] flags 0x%x", i, node->flags);
      case N_BIG /* large/overlow page */:
        err = defrag_fixup_ref(dfc, node_data(node));
        if (unlikely(MDBX_IS_ERROR(err)))
          return err;
        break;
      case N_TREE /* b-tree of named table */:
      case N_TREE | N_DUP /* nested b-tree of multi-values hive */:
        err = defrag_fixup_ref(dfc, ptr_disp(node_data(node), offsetof(tree_t, root)));
        if (unlikely(MDBX_IS_ERROR(err)))
          return err;
        if (err == MDBX_RESULT_TRUE)
          unaligned_poke_u64(1, ptr_disp(node_data(node), offsetof(tree_t, mod_txnid)), dfc->txn->txnid);
        break;
      case 0 /* usual leaf key-value node */:
      case N_DUP /* short sub-page */:
        break;
      }
    }
    break;
  case P_LEAF | P_DUPFIX:
  case P_LARGE:
    break;
  }

  dst->pgno = pgno;
  dst->txnid = dfc->txn->txnid;
  return MDBX_SUCCESS;
}

static int defrag_move(dfc_t *dfc, da_t *arc) {
  MDBX_txn *const txn = dfc->txn;
  ASSERT(arc->key_or_pgno >= NUM_METAS && arc->mapped >= NUM_METAS && arc->npages > 0);
  ASSERT(arc->key_or_pgno + arc->npages <= txn->geo.first_unallocated);
  ASSERT((size_t)arc->mapped + (size_t)arc->npages <= dfc->defrag_edge);
  int err;

  /* Нужно скопировать содержимое страницы в новое место, а также поправить ссылку на перемещённую страницу в
   * родительской. Родительская страница тоже должна быть скопирована/перемещена, вместе с её родительской и так до
   * корня дерева. Корректируемые ссылки точно отсутствуют только у небольшой части страниц, в large/overlow страницах и
   * листьях вложенных b-tree у dupsort-таблиц
   *
   * Поэтому нерационально корректировать ссылки в родительских страницах при перемещении/копировании дочерних, так как
   * это потребует многократного обновления большей части родительских страниц. Напротив, корректировка ссылок внутри
   * страницы при её перемещении/копировании избавляет как от многократного обновления родительских страниц, так и от
   * сканирования в поисках подлежащих корректировке ссылок. Количество поисков в dfc->track при этом не меняется, так
   * как в обоих случаях равно количеству дочерних страниц. Кроме этого, последовательная обработка dfc->track приведет
   * к более упоряченному чтению и записи страниц. */

  const size_t npages = arc->npages;
  const size_t di = ((txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) ? txn_dpl_exist(txn, arc->key_or_pgno) : 0;
  if (di) {
    page_t *dst = txn->wr.dirtylist->items[di].ptr;
    if (!MDBX_AVOID_MSYNC || (txn->flags & MDBX_WRITEMAP)) {
      const page_t *const src = dst;
      dst = pgno2page(txn->env, arc->mapped);
      page_copy(dst, src, pgno2bytes(txn->env, npages));
    }

    /* Seems this is too rare case to make an optimizing dpl_move(txn, from_di, to_pgno) function. */
    txn_dpl_remove_ex(txn, di, npages);
    err = txn_dpl_append(txn, arc->mapped, dst, npages);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    dfc->cycle_pages_moved += npages;
    return defrag_fixup_page(dfc, dst, arc->mapped);
  }

  page_t *const dst =
      ((txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && env_is_page_incore(dfc->txn->env, arc->mapped))
          ? pgno2page(txn->env, arc->mapped)
          : txn->env->page_auxbuf;

  if (env_is_page_incore(dfc->txn->env, arc->key_or_pgno)) {
    const pgr_t pgr = defrag_get_page(dfc, arc->key_or_pgno);
    if (unlikely(pgr.err != MDBX_SUCCESS))
      return pgr.err;
    page_copy(dst, pgr.page, txn->env->ps);
  } else {
#if MDBX_CHECKING > 1
    ASSERT(!pnl_contains(dfc->repnl_clone, arc->key_or_pgno));
#endif /* MDBX_CHECKING > 1 */
    err = osal_pread(txn->env->dxb_mmap.fd, dst, txn->env->ps, pgno2bytes(txn->env, arc->key_or_pgno));
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }

  err = defrag_fixup_page(dfc, dst, arc->mapped);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  if (dst == txn->env->page_auxbuf) {
#if MDBX_CHECKING > 1
    ASSERT(pnl_contains(dfc->repnl_clone, arc->mapped));
#endif /* MDBX_CHECKING > 1 */
    err = osal_pwrite(txn->env->dxb_mmap.fd, dst, txn->env->ps, pgno2bytes(txn->env, arc->mapped));
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }

  DEBUG("moved %u pages %u -> %u", arc->npages, arc->key_or_pgno, arc->mapped);
  dfc->cycle_pages_moved += npages;
  if (unlikely(npages > 1)) {
    MDBX_env *env = txn->env;
    for (pgno_t i = 1; i < npages; ++i) {
      off_t off_src = pgno2bytes(env, arc->key_or_pgno + i);
      off_t off_dst = pgno2bytes(env, arc->mapped + i);
      if ((txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && env_is_page_incore(env, arc->mapped + i)) {
        if (env_is_page_incore(env, arc->key_or_pgno + i))
          memcpy(ptr_disp(env->dxb_mmap.base, off_dst), ptr_disp(env->dxb_mmap.base, off_src), env->ps);
        else {
          err = osal_pread(env->dxb_mmap.fd, ptr_disp(env->dxb_mmap.base, off_dst), env->ps, off_src);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
        }
      } else {
#if MDBX_USE_COPYFILERANGE
        const ssize_t remainted = pgno2bytes(env, npages - i);
        const ssize_t copied = copy_file_range(env->dxb_mmap.fd, &off_src, env->dxb_mmap.fd, &off_dst, remainted, 0);
        err = MDBX_SUCCESS;
        if (unlikely(copied != remainted))
          err = (copied < 0) ? errno : MDBX_EIO;
        break;
#else
#if MDBX_CHECKING > 1
        ASSERT(!pnl_contains(dfc->repnl_clone, bytes2pgno(env, off_src)));
#endif /* MDBX_CHECKING > 1 */
        err = osal_pread(env->dxb_mmap.fd, txn->env->page_auxbuf, env->ps, off_src);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
#if MDBX_CHECKING > 1
        ASSERT(pnl_contains(dfc->repnl_clone, bytes2pgno(env, off_dst)));
#endif /* MDBX_CHECKING > 1 */
        err = osal_pwrite(txn->env->dxb_mmap.fd, txn->env->page_auxbuf, env->ps, off_dst);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
#endif /* MDBX_USE_COPYFILERANGE */
      }
    }
  }

  return MDBX_SUCCESS;
}

static pgno_t defrag_repnl_get(dfc_t *dfc, size_t npages) {
  if (likely(pnl_size(dfc->txn->wr.repnl) >= npages &&
             dfc->defrag_enough >= MDBX_PNL_LEAST(dfc->txn->wr.repnl) + npages)) {
    pgno_t pgno = dfc->largepage_amountleft ? pnl_get_best_sequence(dfc->txn->wr.repnl, npages, dfc->defrag_enough)
                                            : gc_repnl_get_single(dfc->txn);
    ASSERT(pgno == 0 || pgno + npages <= dfc->defrag_enough);
    return pgno;
  }
  return 0;
}

static int defrag_schedule(dfc_t *dfc, da_t *arc, pgno_t assigned) {
  da_t *stack_of_parents[CURSOR_STACK_SIZE * 2];
  size_t depth = 0;
  const size_t npages = arc->npages;

  ASSERT(dfc->remapped_edge < dfc->defrag_enough);
  for (da_t *chain = arc; chain->parent;) {
    chain = dml_search_exact(dfc->arcs, chain->parent);
    ASSERT(chain && chain->npages == 1);
    if (chain->mapped)
      break;

    chain->mapped = defrag_repnl_get(dfc, 1);
    VERBOSE("map%s %u pages %u/%u => %u", "-parent", 1, chain->parent, chain->key_or_pgno, chain->mapped);
    if (!chain->mapped)
      goto bailout;

    ASSERT((size_t)arc->mapped + 1 <= dfc->defrag_enough);
    stack_of_parents[depth++] = chain;
  }

  ASSERT(!arc->mapped);
  if (!assigned)
    assigned = defrag_repnl_get(dfc, npages);
  arc->mapped = assigned;
  VERBOSE("map%s %zu pages %u/%u => %u", "", npages, arc->parent, arc->key_or_pgno, arc->mapped);

  if (arc->mapped) {
    ASSERT(arc->mapped + npages <= dfc->retreat_edge);
    if (npages > 1 && arc->key_or_pgno >= dfc->defrag_enough) {
      ASSERT(dfc->largepage_amountleft >= npages);
      dfc->largepage_amountleft -= npages;
    }
    if (dfc->remapped_edge < arc->mapped)
      dfc->remapped_edge = arc->mapped + npages - 1;
    for (size_t i = 0; i < depth; ++i) {
      da_t *chain = stack_of_parents[i];
      if (dfc->remapped_edge < chain->mapped)
        dfc->remapped_edge = chain->mapped;
    }
    if (likely(dfc->remapped_edge < dfc->retreat_edge)) {
      dfc->cycle_pages_scheduled += npages + depth;
      DEBUG("scheduled %zu+%zu=%zu pages %u -> %u", npages, depth, npages + depth, arc->key_or_pgno, arc->mapped);
      return MDBX_SUCCESS;
    }
  }

bailout:
  VERBOSE("unable find/assign remapping for %zu-pages span, chain-depth %zu, repnl-len %zu, repnl-least %u, detent %u",
          npages, depth, pnl_size(dfc->txn->wr.repnl),
          pnl_size(dfc->txn->wr.repnl) ? MDBX_PNL_LEAST(dfc->txn->wr.repnl) : P_INVALID, dfc->defrag_enough);
  if (pnl_size(dfc->txn->wr.repnl) == 0)
    defrag_stumble(dfc, arc, "are not enough reclaimable pages available in GC", 0, "");
  else if (MDBX_PNL_LEAST(dfc->txn->wr.repnl) + npages >= dfc->defrag_enough)
    defrag_stumble(dfc, arc, "the next reclaimable page ", MDBX_PNL_LEAST(dfc->txn->wr.repnl),
                   " is beyond target edge");
  else {
    ASSERT(npages > 1);
    if ((dfc->stopping_reasons & MDBX_defrag_large_chunk) == 0) {
      dfc->stopping_reasons |= MDBX_defrag_large_chunk;
      defrag_stumble(dfc, arc, "is no span of ", npages, " consecutive/adjacent reclaimable pages");
    }
  }

  /* Не получилось, возвращаем всё выделенное для родительских страниц */
  while (depth > 0) {
    da_t *chain = stack_of_parents[--depth];
    pnl_append_prereserved(dfc->txn->wr.repnl, chain->mapped);
    chain->mapped = 0;
  }

  if (assigned) {
    int err = pnl_append_span(&dfc->txn->wr.repnl, assigned, npages);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }
  arc->mapped = 0;

  pnl_sort(dfc->txn->wr.repnl, dfc->txn->geo.first_unallocated);
  return MDBX_RESULT_TRUE;
}

__hot __noinline static unsigned defrag_move_cost_uncached(dfc_t *dfc, pgno_t pgno, pgno_t span) {
  if (pgno < NUM_METAS || pgno + span >= dfc->retreat_edge)
    return INT_MAX;

  da_t *arc = dml_search_exact(dfc->arcs, pgno);
  if (!arc)
    return pnl_contains(dfc->txn->wr.repnl, pgno) ? 0 : INT_MAX;
  if (arc->mapped)
    return arc->engaged ? INT_MAX : 0;

  size_t npages = arc->npages;
  size_t cost = npages;
  if (npages > 1) {
    if (npages >= span)
      return INT_MAX;
    cost += 64 << ((npages < 12) ? npages : 12);
  }
  while (arc->parent) {
    arc = dml_search_exact(dfc->arcs, arc->parent);
    ASSERT(arc != nullptr);
    if (arc->mapped)
      break;
    cost += arc->key_or_pgno < dfc->retreat_edge;
  }
  return cost;
}

static inline unsigned defrag_move_cost(dfc_t *dfc, pgno_t pgno, pgno_t span) {
  struct cache_item *cache = &dfc->cache_cost[pgno % ARRAY_LENGTH(dfc->cache_cost)];
  if (cache->pgno == pgno)
    return cache->cost;
  cache->pgno = pgno;
  return cache->cost = defrag_move_cost_uncached(dfc, pgno, span);
}

__hot static int defrag_provide_span(dfc_t *const dfc, const size_t npages) {
  ASSERT(npages > 1);
  const pnl_t pnl = dfc->txn->wr.repnl;
  const size_t len = pnl_size(pnl);

  if (len < npages)
    return MDBX_RESULT_TRUE;

  memset(dfc->cache_cost, 0, sizeof(dfc->cache_cost));
  size_t best_begin = MAX_PAGENO, best_cost = len;
#if MDBX_PNL_ASCENDING
#error "FIXME: Since 2026-04-01 alternatives to MDBX_PNL_ASCENDING = 0 are no longer supported."
#else
  for (size_t span, i = len; i > 0 && pnl[i] <= dfc->defrag_enough - npages;) {
    span = 1;
    while (unlikely(MDBX_PNL_CONTIGUOUS(pnl[i - span], pnl[i], span)) &&
           likely((intptr_t)(i - span) > 0 && pnl[i - span] < dfc->defrag_enough))
      ++span;

    pgno_t begin = pnl[i], end = begin + span;
    i -= span;
    size_t cost = 0;
    size_t cost_left = defrag_move_cost(dfc, begin - 1, npages);
    size_t cost_right = defrag_move_cost(dfc, end, npages);
    do {
      if (cost_left < cost_right) {
        cost += cost_left;
        cost_left = defrag_move_cost(dfc, --begin - 1, npages);
      } else if (cost_right < INT_MAX) {
        cost += cost_right;
        cost_right = defrag_move_cost(dfc, ++end, npages);
      } else {
        while (pnl[i] < end && likely(i > 0))
          --i;
        break;
      }
    } while (cost < best_cost && end - begin < npages);

    if (unlikely(end - begin == npages && cost < best_cost)) {
      best_cost = cost;
      best_begin = begin;
      if (!best_cost)
        break;
    }
  }
#endif /* MDBX_PNL_ASCENDING */

  if (/* paranoia */ best_begin + npages >= dfc->retreat_edge || best_begin + npages >= dfc->defrag_edge)
    return MDBX_RESULT_TRUE;

  /* Сначала убираем из repnl выбранные страницы, чтобы не использовать их на перемещение остальных. */
  pnl_clear(dfc->temp);
  pnl_cut_range(pnl, &dfc->temp, best_begin, best_begin + npages);

  /* Теперь сортируем вынутое из repnl и подготавливаем перемещение используемых страниц. */
  const size_t repnl_before = pnl_size(pnl);
  size_t payoff = 0;
  for (pgno_t pgno = best_begin + npages; --pgno >= best_begin;) {
    if (!pnl_contains(dfc->temp, pgno)) {
      da_t *arc = dml_search_exact(dfc->arcs, pgno);
      ASSERT(arc != nullptr);
      if (unlikely(!arc || arc->npages >= npages))
        return MDBX_PROBLEM;
      if (!arc->mapped) {
        int rc = defrag_schedule(dfc, arc, 0);
        if (rc != MDBX_SUCCESS) {
          int err = pnl_append_pnl(&dfc->txn->wr.repnl, dfc->temp);
          ASSERT(!err);
          if (/* paranoia */ unlikely(err != MDBX_SUCCESS))
            return err;
          pnl_sort(dfc->txn->wr.repnl, dfc->txn->geo.first_unallocated);
          return rc;
        }
      } else {
        ASSERT(!arc->engaged);
        payoff += 1;
      }
      arc->engaged = 1;
    }
  }

  int err = pnl_append_pnl(&dfc->lp_reserve, dfc->temp);
  ASSERT(!err);
  if (/* paranoia */ unlikely(err != MDBX_SUCCESS))
    return err;

  const size_t moves = repnl_before - pnl_size(pnl);
  VERBOSE("prepared lp-span %zu, at %zu, %zu moves, %zu repnl", npages, best_begin, moves, pnl_size(dfc->temp));
  ASSERT(pnl_size(dfc->temp) <= npages);
  ASSERT(moves + payoff + pnl_size(dfc->temp) >= npages);
  ASSERT(moves + payoff || npages == pnl_size(dfc->temp));
  (void)payoff;
  return MDBX_SUCCESS;
}

int defrag_cycle(dfc_t *dfc) {
  MDBX_txn *const txn = dfc->txn;
  if (dfc->cycle) {
    dfc->stopping_reasons &= ~(MDBX_defrag_large_chunk | MDBX_defrag_step_size);
    dfc->progress_counter = 0;
    dfc->cycle_pages_scheduled = 0;
    dfc->cycle_pages_moved = 0;
    dfc->cycle_preprogress = 0;
    dfc->cycle += 1;
  }
  if (!defrag_should_continue(dfc, 0))
    return MDBX_RESULT_TRUE;

  int rc = defrag_load_gc(dfc);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  defrag_milestone(dfc);
  rc = txn_basal_update_tbl_roots(txn);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  if (txn->wr.loose_count > 0) {
    rc = gc_merge_loose(txn);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }

  dfc->cycle_initial_score = defrag_score(dfc, dfc->txn->geo.first_unallocated);
  VERBOSE("after-%s: %u => %u (%i), GC-score %" PRIu64, "GC-load", dfc->before_defrag, dfc->txn->geo.first_unallocated,
          dfc->before_defrag - dfc->txn->geo.first_unallocated, dfc->cycle_initial_score);

  /* Проверяем сумму использованных страниц */
  const size_t gc_pages = dfc->gc_tree_pages + dfc->gc_retained_pages;
  const size_t pending_pages =
      txn->wr.loose_count + pnl_size(txn->wr.repnl) + (pnl_size(txn->wr.retired_pages) - /* retired_stored */ 0);
  if (dfc->payload_pages + gc_pages + pending_pages != txn->geo.first_unallocated) {
    ERROR("page usage mismatch (payload %u + gc %zu + pending %zu != allocated %zu), "
          "please use mdbx_chk tool to check DB integrity",
          dfc->payload_pages, gc_pages, pending_pages, (size_t)txn->geo.first_unallocated);
    return LOG_IFERR(MDBX_PROBLEM);
  }

  if (likely(!MDBX_IS_ERROR(rc))) {
    int err = defrag_clear_reclaimed(dfc);
    if (unlikely(err != MDBX_SUCCESS))
      rc = err;
  }
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  defrag_milestone(dfc);
  uint64_t score = defrag_score(dfc, dfc->txn->geo.first_unallocated);
  VERBOSE("after-%s: %u => %u (%i), GC-score %" PRIu64, "GC-clean", dfc->before_defrag, dfc->txn->geo.first_unallocated,
          dfc->before_defrag - dfc->txn->geo.first_unallocated, score);
  if (dfc->payload_pages == 0 || !pnl_size(txn->wr.repnl))
    return defrag_gc_empty(dfc) ? MDBX_RESULT_TRUE : MDBX_LAGGARD_READER;

  if (!dfc->lp_backlog && (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) == 0 && score < dfc->cycle_initial_score) {
    NOTICE("bailout since the GC-score after load and clear 0x%" PRIx64 " is wort than before 0x%" PRIx64, score,
           dfc->cycle_initial_score);
    return MDBX_RESULT_TRUE;
  }

#if MDBX_CHECKING > 1
  dfc->repnl_clone = pnl_clone(txn->wr.repnl);
#endif /* MDBX_CHECKING > 0 */

  if (dfc->payload_items) {
    /* Повторный проход. Все страницы с полезными данными по-прежнему используются, но были перемещены. Отличия могут
     * быть только в страницах связанных с GC. Поэтому корректируем карту по результатам перемещения страниц и удаляем
     * элементы связанные с GC, которые затем будут загружены повторно. */
    da_t *const begin = dfc->arcs->items, *const end = begin + dfc->arcs->length;
    for (da_t *i = begin; i != end; ++i) {
      /* Первым проходом корректируем ссылки на родительские страницы по результатам перемещения, иначе после изменения
       * основных номеров связь будет потеряна. */
      if (likely(i->parent)) {
        da_t *p = dml_search_exact(dfc->arcs, i->parent);
        ASSERT(p != nullptr && p->key_or_pgno == i->parent);
        i->parent = p->mapped ? p->mapped : p->key_or_pgno;
      }
    }

    dfc->remapped_edge = 0;
    dfc->largepage_count = 0;
    dfc->largepage_amountleft = 0;
    dfc->largepage_max = 0;
    da_t *w = begin;
    for (da_t *r = begin; r != end; ++r) {
      /* Корректируем номера страниц по результатам перемещения и удаляем/пропускаем элементы связанные с GC. */
      if (!r->gc) {
        if (r->mapped) {
          if (dfc->stumble_pgno == r->key_or_pgno) {
            dfc->stumble_pgno = 0;
            dfc->stumble_span = 0;
          }
          if (dfc->lp_backlog == r->key_or_pgno)
            dfc->lp_backlog = 0;
          r->key_or_pgno = r->mapped;
          r->mapped = 0;
          r->engaged = 0;
        }
        *w = *r;
        const size_t npages = w->npages;
        if (npages > 1 && w->key_or_pgno >= dfc->defrag_enough) {
          dfc->largepage_count += 1;
          dfc->largepage_amountleft += npages;
          if (npages > dfc->largepage_max)
            dfc->largepage_max = npages;
        }
        ++w;
      }
    }
    dfc->arcs->length = w - begin;
    if (unlikely(dfc->arcs->length != dfc->payload_items)) {
      ERROR("mismatch entries (%zu != expected %zu) after filter-out GC on next pass", dfc->arcs->length,
            dfc->payload_items);
      return MDBX_PROBLEM;
    }
  } else {
    env_clear_incore_cache(txn->env);
  }

  /* Обходим b-tree и сохраняем в dfc->arcs[] номера страниц больше dfc->walk_cutoff, вместе с их родительскими
   * страницами на всем пути от корня b-tree. */
  rc = walk_pages(txn, defrag_walker, dfc, dfc->payload_items ? dont_walk_MAIN : dont_check_keys_ordering);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;
  if (dfc->arcs->length < 1)
    return MDBX_SUCCESS;

  if (!dfc->cycle) {
    defrag_milestone(dfc);
    dfc->progress_counter = 0;
    dfc->cycle = 1;
  }
  dfc->cycle_preprogress = dfc->progress_counter;
  defrag_milestone(dfc);
  ASSERT(dfc->cycle_pages_scheduled == 0 && dfc->cycle_pages_moved == 0);

  /* После сортировки в начале массива dfc->arcs[] располагается некоторое количество пар целевая-родительская
   * номеров страниц, из которых необходимо переместить данные. Перемещать содержимое страниц есть смысл пока
   * выполняются два условия:
   *  - очередная исходная страница с наибольшим номером прилегает к концу БД (к границе распределенных страниц);
   *  - очередная целевая переработанная страница ближе к началу БД чем исходная.
   *
   * Однако, некоторые сложности возникают из-за large/overflow-страниц, для которых необходимо выделять/искать
   * последовательности свободных страниц. Соответственно, такие последовательности требуется экономить (не расходовать
   * на одиночные страницы или на интервалы меньшего размера). Поэтому при наличии хотя-бы одной large/overflow-страницы
   * размером более одной страницы выделение выполняется с поиском наиболее подходящей по размеру последовательности,
   * включая одиночные.
   *
   * Далее, перемещение длинных large/overflow-страниц может потребовать предварительного формирования соответствующих
   * последовательностей свободных страниц, т.е. предварительного перемещения других страниц внутри теля БД, которые не
   * требуют перемещения.
   *
   * Далее, перемещение данных потребует корректировки ссылок в родительских страницах по цепочки от листовых до корня.
   * Соответственно, нет смысла записывать какие-либо родительские страницы до завершения создания карты перемещений. */

  dml_sort(dfc->arcs);
  pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated);
  const pgno_t most_retired = pnl_size(txn->wr.retired_pages) ? MDBX_PNL_MOST(txn->wr.retired_pages) : 0;
  ASSERT(MDBX_PNL_MOST(txn->wr.repnl) < txn->geo.first_unallocated - 1);
  dfc->retreat_edge = dfc->defrag_enough;
  dfc->defrag_edge = txn->geo.first_unallocated;
  da_t *const begin = dfc->arcs->items, *const end = begin + dfc->arcs->length;

  if (dfc->largepage_count > 0) {
    /* Есть large/overflow-страницы длиной больше 1, для которых требует много дополнительных действий */
    if (!dfc->lp_reserve) {
      /* Резервирование выполняется один раз, на все последующие циклы */
      dfc->lp_reserve = pnl_alloc(dfc->largepage_amountleft);
      dfc->temp = pnl_alloc(dfc->largepage_max);
      if (!dfc->lp_reserve || !dfc->temp)
        return MDBX_ENOMEM;
    }
  }

  da_t *lp = nullptr;
  if (dfc->lp_backlog) {
    lp = dml_search_exact(dfc->arcs, dfc->lp_backlog);
    ASSERT(lp != nullptr);
    if (lp) {
      ASSERT(lp->npages > 1);
      pgno_t pgno = pnl_get_best_sequence(dfc->txn->wr.repnl, lp->npages, dfc->defrag_enough);
      if (pgno) {
        int err = defrag_schedule(dfc, lp, pgno);
        if (err == MDBX_SUCCESS)
          lp = nullptr;
        else if (err != MDBX_RESULT_TRUE)
          return err;
      }
    }
  }

  for (da_t *i = begin; !defrag_discontinued(dfc) && i != end && i->key_or_pgno >= dfc->retreat_edge && !lp; ++i) {
    if (i->key_or_pgno < dfc->remapped_edge) {
      defrag_stumble(dfc, i,
                     "all the unused pages suitable for defragmentation have been used up, the re-mapped edge is ",
                     dfc->remapped_edge, "");
      break;
    }

    if (dfc->defrag_edge - i->npages != i->key_or_pgno) {
      const pgno_t tail = dfc->defrag_edge - 1;
      if (most_retired == tail) {
        defrag_stumble(dfc, i, "the adjacent page ", most_retired, " is in the retired list");
        break;
      }
      if (pnl_size(txn->wr.repnl) && MDBX_PNL_LEAST(txn->wr.repnl) == tail) {
        defrag_stumble(dfc, i, "the available reclaimed pages are beyond the defragmentation tail", 0, "");
        break;
      }
      int err = defrag_gc_lookup_page(dfc, dfc->defrag_edge - 1, &dfc->stopor);
      if (unlikely(err != MDBX_SUCCESS)) {
        ERROR("unexpected missing page %" PRIaPGNO, tail);
        return (err == MDBX_NOTFOUND) ? MDBX_PROBLEM : err;
      }
      defrag_stumble(dfc, i, "the gap-page is located in the ", dfc->stopor, " GC-entry which is used/read");
      break;
    }

    if (!i->mapped) {
      int err = defrag_schedule(dfc, i, 0);
      if (unlikely(err != MDBX_SUCCESS)) {
        ASSERT((i->mapped != 0) == (err == MDBX_SUCCESS));
        if (unlikely(err != MDBX_RESULT_TRUE))
          return err;
        lp = (i->npages > 1) ? i : nullptr;
        break;
      }
    }
    VERBOSE("turn-%s %+i, %u => %u", "move", i->npages, dfc->defrag_edge, i->key_or_pgno);
    dfc->defrag_edge = i->key_or_pgno;

    if (pnl_size(txn->wr.repnl) > 0 && MDBX_PNL_MOST(txn->wr.repnl) == dfc->defrag_edge - 1) {
      const pgno_t npages = pnl_crop_tail_sequence(txn->wr.repnl);
      rc = pnl_append_span(&txn->wr.retired_pages, dfc->defrag_edge - npages, npages);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      VERBOSE("turn-%s %+i, %u => %u", "crop", npages, dfc->defrag_edge, dfc->defrag_edge - npages);
      dfc->defrag_edge -= npages;
    }
    defrag_progress(dfc, i->npages);
    if (dfc->cycle_pages_scheduled >= dfc->move_batch_size) {
      dfc->stopping_reasons |= MDBX_defrag_step_size;
      break;
    }
  }

  if (lp || dfc->stopping_reasons == MDBX_defrag_large_chunk) {
    /* Для быстрой дефрагментации требуется использовать имеющиеся последовательности свободных страниц максимально
     * эффективно:
     *  - с одной стороны, нет смысла перемещать ближние куски раньше дальних, иначе дефрагментация всё равно
     *    остановится на не перемещённых дальних;
     *  - с другой стороны, выделяя последовательности для коротких кусков, можно потерять возможность
     *    выделить и/или сформировать последовательности для последующих более длинных.
     *
     * По результатам серии экспериментов, лучше всех оказался такой подход:
     *  1. Планируем перемещение регулярных/одиночных страниц и large/overflow-страниц, двигаясь от конца БД к началу,
     *     пока удаётся выделять последовательности.
     *  2. Если останавливаемся из-за отсутствия свободной последовательности необходимой длины, то
     *     прерываем основное регулярный цикл перемещения и далее в текущем цикле дефрагментации занимается
     *     только large/overflow-страницами.
     *  3. Для каждой large/overflow-странице пытаемся найти наиболее подходящую последовательность свободных страниц,
     *     а при отсутствии подготавливаем интервал свободных страниц для перемещения на следующем цикле. */
    defrag_milestone(dfc);
    for (da_t *i = lp ? lp : begin;
         !defrag_discontinued(dfc) && i != end && dfc->largepage_amountleft &&
         pnl_size(dfc->txn->wr.repnl) > dfc->largepage_amountleft && i->key_or_pgno > dfc->remapped_edge &&
         i->key_or_pgno > MDBX_PNL_LEAST(dfc->txn->wr.repnl);
         defrag_milestone(dfc), ++i) {
      if (!i->mapped && i->npages > 1) {
        ASSERT(dfc->largepage_amountleft >= i->npages);
        if (i != lp) {
          pgno_t pgno = pnl_get_best_sequence(dfc->txn->wr.repnl, i->npages, dfc->defrag_enough);
          if (pgno) {
            int err = defrag_schedule(dfc, i, pgno);
            if (unlikely(err != MDBX_SUCCESS)) {
              if (err == MDBX_RESULT_TRUE)
                break;
              return err;
            }
          }
        }
        if (!i->mapped) {
          int err = defrag_provide_span(dfc, i->npages);
          if (err != MDBX_SUCCESS) {
            if (unlikely(err != MDBX_RESULT_TRUE))
              return err;
            break;
          }
          dfc->largepage_amountleft -= i->npages;
          dfc->lp_backlog = (dfc->lp_backlog < i->key_or_pgno) ? i->key_or_pgno : dfc->lp_backlog;
        }
        defrag_progress(dfc, i->npages);
        if (dfc->cycle_pages_scheduled >= dfc->move_batch_size) {
          dfc->stopping_reasons |= MDBX_defrag_step_size;
          break;
        }
      }
    }
  }

  score = defrag_score(dfc, dfc->defrag_edge);
  defrag_milestone(dfc);
  VERBOSE("after-%s: %u => %u (%i), GC-score %" PRIu64, "tree-sift", dfc->before_defrag, dfc->defrag_edge,
          dfc->before_defrag - dfc->defrag_edge, score);
  if (dfc->lp_reserve) {
    rc = pnl_append_pnl(&txn->wr.retired_pages, dfc->lp_reserve);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    pnl_setsize(dfc->lp_reserve, 0);
  }
  if (/* paranoia */ txn->flags & MDBX_TXN_ERROR)
    return MDBX_EIO;
  for (da_t *i = begin; i != end; ++i) {
    if (i->mapped) {
      ASSERT(i->mapped < dfc->defrag_edge);
      rc = defrag_move(dfc, i);
      if (unlikely(rc != MDBX_SUCCESS))
        break;
      rc = pnl_append_span(&txn->wr.retired_pages, i->key_or_pgno, i->npages);
      if (unlikely(rc != MDBX_SUCCESS))
        break;
    }
    defrag_progress(dfc, i->npages);
    if (defrag_aborted(dfc)) {
      rc = MDBX_RESULT_TRUE;
      break;
    }
  }

  defrag_milestone(dfc);
  if (rc != MDBX_SUCCESS) {
    txn->flags |= MDBX_TXN_ERROR;
    return rc;
  }

  ASSERT(dfc->cycle_pages_scheduled == dfc->cycle_pages_moved);
  if (dfc->cycle_pages_moved) {
    dfc->total_pages_moved += dfc->cycle_pages_moved;
    TXN_FOREACH_DBI_ALL(txn, dbi) {
      if ((txn->dbi_state[dbi] & (DBI_VALID | DBI_LINDO | DBI_STALE)) == (DBI_VALID | DBI_LINDO) &&
          txn->dbs[dbi].root != P_INVALID) {
        da_t *dm = dml_search_exact(dfc->arcs, txn->dbs[dbi].root);
        if (dm && dm->mapped != 0) {
          txn->dbs[dbi].root = dm->mapped;
          txn->dbi_state[dbi] |= DBI_DIRTY;
          txn->flags |= MDBX_TXN_DIRTY;
        }
      }
    }
  } else {
    ASSERT(dfc->defrag_edge == txn->geo.first_unallocated);
    ASSERT(dfc->cycle_pages_scheduled == 0);
  }

  score = defrag_score(dfc, dfc->txn->geo.first_unallocated);
  defrag_milestone(dfc);
  VERBOSE("after-%s: %u => %u (%i), GC-score %" PRIu64, "move-pages", dfc->before_defrag, txn->geo.first_unallocated,
          dfc->before_defrag - txn->geo.first_unallocated, score);

  return (dfc->stumble_pgno || (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) || score > dfc->cycle_initial_score)
             ? MDBX_SUCCESS
             : MDBX_RESULT_TRUE;
}

void defrag_destroy(dfc_t *dfc) {
  dml_free(&dfc->arcs);
  pnl_free(dfc->temp);
  dfc->temp = nullptr;
  pnl_free(dfc->lp_reserve);
  dfc->lp_reserve = nullptr;
}

int defrag_init(dfc_t *dfc, MDBX_txn *txn, size_t defrag_atleast_pages, size_t spend_atleast_wallclock_dot16,
                size_t defrag_enough_pages, size_t limit_spend_wallclock_dot16, intptr_t preferred_move_batch_size) {
  memset(dfc, 0, sizeof(*dfc));
  dfc->start_timestamp = osal_monotime();
  if (preferred_move_batch_size < 0) {
    preferred_move_batch_size = 0;
    if (limit_spend_wallclock_dot16) {
      /* TODO: подстроить размер (возможно динамически на каждом цикле) так, чтобы время записи порции не превышало
       * 5-10% от таймаута. */
      preferred_move_batch_size = bytes2pgno(txn->env, 64 * MEGABYTE);
    }
  }
  dfc->move_batch_size = (preferred_move_batch_size && (size_t)preferred_move_batch_size < MAX_PAGENO)
                             ? (size_t)preferred_move_batch_size
                             : MAX_PAGENO;

  dfc->txn = txn;
  dfc->before_defrag = txn->geo.first_unallocated;
  dfc->last_allocated = dfc->before_defrag;
  if (limit_spend_wallclock_dot16)
    dfc->wallclock_detent = dfc->start_timestamp + osal_16dot16_to_monotime(limit_spend_wallclock_dot16);
  if (spend_atleast_wallclock_dot16)
    dfc->wallclock_atleast = dfc->start_timestamp + osal_16dot16_to_monotime(spend_atleast_wallclock_dot16);

  /* Загружаем информацию о всех таблицах. */
  MDBX_stat stat;
  int err = tbl_stat_summary(txn, &stat);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  dfc->payload_pages =
      NUM_METAS + (size_t)stat.ms_branch_pages + (size_t)stat.ms_leaf_pages + (size_t)stat.ms_overflow_pages;
  dfc->largepage_max = /* устанавливаем в 1 как признак наличия overload/large-страниц */ stat.ms_overflow_pages > 0;
  dfc->summary_depth = stat.ms_depth;

  /* По количеству используемых и выделенных страниц, оцениваем сколько можем дефрагментировать/освободить. */
  const size_t max_defrag = txn->geo.first_unallocated - dfc->payload_pages;
  if (txn->geo.first_unallocated < dfc->payload_pages)
    return MDBX_PROBLEM;
  if (unlikely(max_defrag == 0))
    return MDBX_SUCCESS;

  err = pnl_reserve(&txn->wr.repnl, max_defrag);
  if (unlikely(err != MDBX_SUCCESS))
    return err;
  err = pnl_reserve(&txn->wr.retired_pages, max_defrag);
  if (unlikely(err != MDBX_SUCCESS))
    return err;
  if (txn->dbs[FREE_DBI].items) {
    err = rkl_reserve(&txn->wr.gc.reclaimed, txn->dbs[FREE_DBI].items);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }

  dfc->defrag_atleast = txn->geo.first_unallocated;
  if (defrag_atleast_pages)
    dfc->defrag_atleast =
        (defrag_atleast_pages < max_defrag) ? txn->geo.first_unallocated - defrag_atleast_pages : dfc->payload_pages;
  dfc->defrag_enough = dfc->payload_pages;
  if (defrag_enough_pages && defrag_enough_pages < max_defrag)
    dfc->defrag_enough = txn->geo.first_unallocated - defrag_enough_pages;

  /* Необходимо найти в структуре дерева страницы расположенные близко к концу файла БД, а затем переместить их
   * содержимое в страницы ближе к началу БД. При этом для каждой дочерней страницы придеться делать также копию её
   * родительской. Поэтому, при поиске страниц, необходимо запоминать всю цепочку страниц, от корня до каждой целевой.
   * Удобнее всего для этого использовать граф в форме набора пар/дуг из номеров страниц дочерняя-родительская. */
  size_t arcs_needed;
  if (!dfc->largepage_max) {
    /* Если нет large/overflow страниц, то необходимо отследить только подлежащие перемещению и их родительские
     * страницы в структуре дерева. Количество перемещаемых страниц точно известно, а количество родительских связей
     * можно оценить/ограничить сверху через общую высоту деревьев и количество страниц. */
    size_t extra_for_parents = (size_t)(((uint64_t)dfc->summary_depth) * max_defrag / dfc->payload_pages);
    arcs_needed = max_defrag + extra_for_parents;
    if (arcs_needed > dfc->payload_pages)
      arcs_needed = dfc->payload_pages;
    dfc->walk_cutoff = dfc->payload_pages;
  } else {
    /* Если есть large/overflow страницы, то для их перемещения потребуются последовательности смежных свободных
     * страниц. В свою очередь, может потребоваться перемещение страниц внутри основного тела БД для формирования
     * таких последовательностей для чего потребуется информация о всех страницах с полезными данными. */
    arcs_needed = dfc->payload_pages;
    dfc->walk_cutoff = NUM_METAS;
  }
  dfc->arcs = dml_alloc(arcs_needed);
  if (unlikely(!dfc->arcs))
    return MDBX_ENOMEM;

  const size_t lo = (dfc->payload_pages < max_defrag) ? dfc->payload_pages : max_defrag;
  const size_t hi = (dfc->payload_pages > max_defrag) ? dfc->payload_pages : max_defrag;
  dfc->notify_watchmask = 7;
  while (dfc->notify_watchmask < (hi >> 10) && dfc->notify_watchmask < (lo >> 7) && dfc->notify_watchmask < 500)
    dfc->notify_watchmask += dfc->notify_watchmask + 1;

  return defrag_should_continue(dfc, 0) ? MDBX_SUCCESS : MDBX_RESULT_TRUE;
}

static inline size_t dml_size2bytes(ptrdiff_t size) {
  ASSERT(size >= CURSOR_STACK_SIZE && (size_t)size <= PAGELIST_LIMIT);
#if MDBX_DML_PREALLOC_FOR_RADIXSORT
  size += size;
#endif /* MDBX_DML_PREALLOC_FOR_RADIXSORT */
  STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dml_t) +
                    (PAGELIST_LIMIT * (MDBX_DML_PREALLOC_FOR_RADIXSORT + 1)) * sizeof(da_t) +
                    MDBX_PNL_GRANULATE * sizeof(void *) * 2 <
                SIZE_MAX / 4 * 3);
  size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dml_t) + size * sizeof(da_t),
                               MDBX_PNL_GRANULATE * sizeof(void *) * 2) -
                 MDBX_ASSUME_MALLOC_OVERHEAD;
  return bytes;
}

static inline size_t dml_bytes2size(const ptrdiff_t bytes) {
  size_t size = (bytes - sizeof(dml_t)) / sizeof(da_t);
#if MDBX_DML_PREALLOC_FOR_RADIXSORT
  size >>= 1;
#endif /* MDBX_DML_PREALLOC_FOR_RADIXSORT */
  ASSERT(size > CURSOR_STACK_SIZE && size <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
  return size;
}

__cold dml_t *dml_alloc(size_t size) {
  size_t bytes = dml_size2bytes((size < PAGELIST_LIMIT) ? ((size > CURSOR_STACK_SIZE) ? size : CURSOR_STACK_SIZE)
                                                        : PAGELIST_LIMIT);
  dml_t *const dml = osal_malloc(bytes);
  if (likely(dml)) {
    dml->length = 0;
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(dml);
#endif /* osal_malloc_usable_size */
    dml->limit = dml_bytes2size(bytes);
    ASSERT(dml->limit >= size);
  }
  return dml;
}

static __cold dml_t *dml_extend(dml_t *dml) {
  if (!dml)
    return dml_alloc(MDBX_PNL_INITIAL);
  if (unlikely(dml->limit == PAGELIST_LIMIT))
    return nullptr;
  size_t size = (dml->limit < MDBX_PNL_INITIAL * 42) ? dml->limit + dml->limit : dml->limit + dml->limit / 2;
  size_t bytes = dml_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT);
  dml = osal_realloc(dml, bytes);
  if (likely(dml)) {
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(dml);
#endif /* osal_malloc_usable_size */
    dml->limit = dml_bytes2size(bytes);
  }
  return dml;
}

da_t *dml_append(dml_t **pdml, pgno_t key) {
  dml_t *dml = *pdml;
  ASSERT(!dml || dml->length <= dml->limit);
  if (unlikely(!dml || dml->length == dml->limit)) {
    dml = dml_extend(dml);
    if (unlikely(!dml))
      return nullptr;
    *pdml = dml;
  }
  ASSERT(dml->length < dml->limit);

  da_t *dm = &dml->items[dml->length++], init = {.key_or_pgno = key};
  *dm = init;
  return dm;
}

#define MDBX_DML_EXTRACT_KEY(ptr) (P_INVALID - (ptr)->key_or_pgno)
RADIXSORT_IMPL(dm, da_t, MDBX_DML_EXTRACT_KEY, MDBX_DML_PREALLOC_FOR_RADIXSORT, 0)

#define DM_SORT_CMP(first, last) ((first).key_or_pgno > (last).key_or_pgno)
SORT_IMPL(dm_sort, false, da_t, DM_SORT_CMP)

void dml_sort(dml_t *dml) {
  if (likely(dml->length < MDBX_RADIXSORT_THRESHOLD) || unlikely(!dm_radixsort(dml->items, dml->length)))
    dm_sort(dml->items, dml->items + dml->length);
}

#define DM_SEARCH_CMP(dm, page) ((dm).key_or_pgno > (page))
SEARCH_IMPL(dm_bsearch, da_t, pgno_t, DM_SEARCH_CMP)

da_t *dml_search(dml_t *dml, pgno_t pgno) {
  const da_t *const begin = dml->items;
  const da_t *const it = dm_bsearch(begin, dml->length, pgno);
  const da_t *const end = begin + dml->length;
  ASSERT(it >= begin && it <= end);
  if (it != begin)
    ASSERT(DM_SEARCH_CMP(it[-1], pgno));
  if (it != end)
    ASSERT(!DM_SEARCH_CMP(it[0], pgno));
  return (da_t *)it;
}

static inline void dpl_init(dpl_t *dl) {
  static const page_t dpl_stub_pageB = {INVALID_TXNID,
                                        0,
                                        P_BAD,
                                        {0},
                                        /* pgno */ 0};
  ASSERT(dpl_stub_pageB.flags == P_BAD && dpl_stub_pageB.pgno == 0);
  dl->pages_including_loose = 0;
  dl->items[0].ptr = (page_t *)&dpl_stub_pageB;
  dl->items[0].pgno = 0;
#if MDBX_DPL_CACHE_NPAGES
  dl->items[0].npages = 1;
#endif /* MDBX_DPL_CACHE_NPAGES */
  dl->sorted = dpl_setlen(dl, 0);
  ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
}

static inline size_t dpl_size2bytes(ptrdiff_t size) {
  ASSERT(size >= CURSOR_STACK_SIZE && (size_t)size <= PAGELIST_LIMIT);
#if MDBX_DPL_PREALLOC_FOR_RADIXSORT
  size += size;
#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
  STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) +
                    (PAGELIST_LIMIT * (MDBX_DPL_PREALLOC_FOR_RADIXSORT + 1)) * sizeof(dp_t) +
                    MDBX_PNL_GRANULATE * sizeof(void *) * 2 <
                SIZE_MAX / 4 * 3);
  size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) + size * sizeof(dp_t),
                               MDBX_PNL_GRANULATE * sizeof(void *) * 2) -
                 MDBX_ASSUME_MALLOC_OVERHEAD;
  return bytes;
}

static inline size_t dpl_bytes2size(const ptrdiff_t bytes) {
  size_t size = (bytes - sizeof(dpl_t)) / sizeof(dp_t);
#if MDBX_DPL_PREALLOC_FOR_RADIXSORT
  size >>= 1;
#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
  ASSERT(size > CURSOR_STACK_SIZE && size <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
  return size;
}

void txn_dpl_free(MDBX_txn *txn) {
  if (likely(txn->wr.dirtylist)) {
    txn_dpl_clear(txn);
    osal_free(txn->wr.dirtylist);
    txn->wr.dirtylist = nullptr;
  }
}

dpl_t *txn_dpl_reserve(MDBX_txn *txn, size_t size) {
  cASSERT0(txn, (txn->flags & txn_ro_flat) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  size_t bytes = dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT);
  dpl_t *const dl = osal_realloc(txn->wr.dirtylist, bytes);
  if (likely(dl)) {
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(dl);
#endif /* osal_malloc_usable_size */
    dl->detent = dpl_bytes2size(bytes);
    cASSERT0(txn, txn->wr.dirtylist == nullptr || dl->length <= dl->detent);
    txn->wr.dirtylist = dl;
  }
  return dl;
}

int txn_dpl_alloc(MDBX_txn *txn) {
  cASSERT0(txn, (txn->flags & txn_ro_flat) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
  const size_t wanna =
      ((txn->flags & txn_ro_nested) == 0)
          ? ((txn->env->options.dp_initial < txn->geo.upper) ? txn->env->options.dp_initial : txn->geo.upper)
          : /* minimal */ CURSOR_STACK_SIZE;
#if MDBX_CHECKING > 0
  if (txn->wr.dirtylist)
    /* обнуляем чтобы не сработал ассерт внутри dpl_reserve() */
    txn->wr.dirtylist->sorted = txn->wr.dirtylist->length = 0;
#endif /* MDBX_CHECKING > 0 */
  if (unlikely(!txn->wr.dirtylist || txn->wr.dirtylist->detent < wanna || txn->wr.dirtylist->detent > wanna + wanna) &&
      unlikely(!txn_dpl_reserve(txn, wanna)))
    return MDBX_ENOMEM;

  /* LY: wr.dirtylist не может быть nullptr, так как либо уже выделен, либо будет выделен в dpl_reserve(). */
  /* coverity[FORWARD_NULL] */
  dpl_init(txn->wr.dirtylist);
  return MDBX_SUCCESS;
}

#define MDBX_DPL_EXTRACT_KEY(ptr) ((ptr)->pgno)
RADIXSORT_IMPL(dp, dp_t, MDBX_DPL_EXTRACT_KEY, MDBX_DPL_PREALLOC_FOR_RADIXSORT, 1)

#define DP_SORT_CMP(first, last) ((first).pgno < (last).pgno)
SORT_IMPL(dp_sort, false, dp_t, DP_SORT_CMP)

__hot __noinline dpl_t *txn_dpl_sort_slowpath(const MDBX_txn *txn) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  dpl_t *dl = txn->wr.dirtylist;
  ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  const size_t unsorted = dl->length - dl->sorted;
  if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || unlikely(!dp_radixsort(dl->items + 1, dl->length))) {
    if (dl->sorted > unsorted / 4 + 4 &&
        (MDBX_DPL_PREALLOC_FOR_RADIXSORT || dl->length + unsorted < dl->detent + dpl_gap_mergesort)) {
      dp_t *const sorted_begin = dl->items + 1;
      dp_t *const sorted_end = sorted_begin + dl->sorted;
      dp_t *const end =
          dl->items + (MDBX_DPL_PREALLOC_FOR_RADIXSORT ? dl->length + dl->length + 1 : dl->detent + dpl_reserve_gap);
      dp_t *const tmp = end - unsorted;
      ASSERT(dl->items + dl->length + 1 < tmp);
      /* copy unsorted to the end of allocated space and sort it */
      memcpy(tmp, sorted_end, unsorted * sizeof(dp_t));
      dp_sort(tmp, tmp + unsorted);
      /* merge two parts from end to begin */
      dp_t *__restrict w = dl->items + dl->length;
      dp_t *__restrict l = dl->items + dl->sorted;
      dp_t *__restrict r = end - 1;
      do {
        const bool cmp = expect_with_probability(l->pgno > r->pgno, 0, .5);
#if defined(__LCC__) || __CLANG_PREREQ(13, 0) || !MDBX_HAVE_CMOV
        *w = cmp ? *l-- : *r--;
#else
        *w = cmp ? *l : *r;
        l -= cmp;
        r += (ptrdiff_t)cmp - 1;
#endif
      } while (likely(--w > l));
      ASSERT(r == tmp - 1);
      ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
      if (CHECKS1_ENABLED())
        for (size_t i = 0; i <= dl->length; ++i)
          ASSERT(dl->items[i].pgno < dl->items[i + 1].pgno);
    } else {
      dp_sort(dl->items + 1, dl->items + dl->length + 1);
      ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
    }
  } else {
    ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  }
  dl->sorted = dl->length;
  return dl;
}

/* Returns the index of the first dirty-page whose pgno
 * member is greater than or equal to id. */
#define DP_SEARCH_CMP(dp, id) ((dp).pgno < (id))
SEARCH_IMPL(dp_bsearch, dp_t, pgno_t, DP_SEARCH_CMP)

__hot __noinline size_t txn_dpl_search(const MDBX_txn *txn, pgno_t pgno) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  dpl_t *dl = txn->wr.dirtylist;
  ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  if (CHECKS2_ENABLED()) {
    for (const dp_t *ptr = dl->items + dl->sorted; --ptr > dl->items;) {
      ASSERT(ptr[0].pgno < ptr[1].pgno);
      ASSERT(ptr[0].pgno >= NUM_METAS);
    }
  }

  switch (dl->length - dl->sorted) {
  default:
    /* sort a whole */
    txn_dpl_sort_slowpath(txn);
    break;
  case 0:
    /* whole sorted cases */
    break;

#define LINEAR_SEARCH_CASE(N)                                                                                          \
  case N:                                                                                                              \
    if (dl->items[dl->length - N + 1].pgno == pgno)                                                                    \
      return dl->length - N + 1;                                                                                       \
    __fallthrough

    /* use linear scan until the threshold */
    LINEAR_SEARCH_CASE(7); /* fall through */
    LINEAR_SEARCH_CASE(6); /* fall through */
    LINEAR_SEARCH_CASE(5); /* fall through */
    LINEAR_SEARCH_CASE(4); /* fall through */
    LINEAR_SEARCH_CASE(3); /* fall through */
    LINEAR_SEARCH_CASE(2); /* fall through */
  case 1:
    if (dl->items[dl->length].pgno == pgno)
      return dl->length;
    /* continue bsearch on the sorted part */
    break;
  }
  return dp_bsearch(dl->items + 1, dl->sorted, pgno) - dl->items;
}

const page_t *debug_txn_dpl_find(const MDBX_txn *txn, const pgno_t pgno) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  const dpl_t *dl = txn->wr.dirtylist;
  if (dl) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
    ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
    for (size_t i = dl->length; i > dl->sorted; --i)
      if (dl->items[i].pgno == pgno)
        return dl->items[i].ptr;

    if (dl->sorted) {
      const size_t i = dp_bsearch(dl->items + 1, dl->sorted, pgno) - dl->items;
      if (dl->items[i].pgno == pgno)
        return dl->items[i].ptr;
    }
  } else {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
  }
  return nullptr;
}

void txn_dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  dpl_t *dl = txn->wr.dirtylist;
  ASSERT((intptr_t)i > 0 && i <= dl->length);
  ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  dl->pages_including_loose -= npages;
  dl->sorted -= dl->sorted >= i;
  dl->length -= 1;
  memmove(dl->items + i, dl->items + i + 1, (dl->length - i + 2) * sizeof(dl->items[0]));
  ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
}

int __must_check_result txn_dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
#if MDBX_DPL_CACHE_NPAGES
  const dp_t dp = {page, pgno, (pgno_t)npages};
#else
  const dp_t dp = {page, pgno};
#endif /* MDBX_DPL_CACHE_NPAGES */
  if ((txn->flags & MDBX_WRITEMAP) == 0) {
    size_t *const ptr = ptr_disp(page, -(ptrdiff_t)sizeof(size_t));
    *ptr = txn->wr.dirtylru;
  }

  dpl_t *dl = txn->wr.dirtylist;
  cASSERT0(txn, dl->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
  cASSERT0(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  if (CHECKS2_ENABLED()) {
    for (size_t i = dl->length; i > 0; --i) {
      ASSERT(dl->items[i].pgno != dp.pgno);
      if (unlikely(dl->items[i].pgno == dp.pgno)) {
        ERROR("Page %u already exist in the DPL at %zu", dp.pgno, i);
        return MDBX_PROBLEM;
      }
    }
  }

  if (unlikely(dl->length == dl->detent)) {
    if (unlikely(dl->detent >= PAGELIST_LIMIT)) {
      ERROR("DPL is full (PAGELIST_LIMIT %zu)", PAGELIST_LIMIT);
      return MDBX_TXN_FULL;
    }
    const size_t size = (dl->detent < MDBX_PNL_INITIAL * 42) ? dl->detent + dl->detent : dl->detent + dl->detent / 2;
    dl = txn_dpl_reserve(txn, size);
    if (unlikely(!dl))
      return MDBX_ENOMEM;
    cASSERT0(txn, dl->length < dl->detent);
  }

  /* Сортировка нужна для быстрого поиска, используем несколько тактик:
   *  1) Сохраняем упорядоченность при естественной вставке в нужном порядке.
   *  2) Добавляем в не-сортированный хвост, который сортируем и сливаем
   *     с отсортированной головой по необходимости, а пока хвост короткий
   *     ищем в нём сканированием, избегая большой пересортировки.
   *  3) Если не-сортированный хвост короткий, а добавляемый элемент близок
   *     к концу отсортированной головы, то выгоднее сразу вставить элемент
   *     в нужное место.
   *
   * Алгоритмически:
   *  - добавлять в не-сортированный хвост следует только если вставка сильно
   *    дорогая, т.е. если целевая позиция элемента сильно далека от конца;
   *  - для быстрой проверки достаточно сравнить добавляемый элемент с отстоящим
   *    от конца на максимально-приемлемое расстояние;
   *  - если список короче, либо элемент в этой позиции меньше вставляемого,
   *    то следует перемещать элементы и вставлять в отсортированную голову;
   *  - если не-сортированный хвост длиннее, либо элемент в этой позиции больше,
   *    то следует добавлять в не-сортированный хвост. */

  dl->pages_including_loose += npages;
  dp_t *i = dl->items + dl->length;

  const ptrdiff_t pivot = (ptrdiff_t)dl->length - dpl_insertion_threshold;
#if MDBX_HAVE_CMOV
  const pgno_t pivot_pgno =
      dl->items[(dl->length < dpl_insertion_threshold) ? 0 : dl->length - dpl_insertion_threshold].pgno;
#endif /* MDBX_HAVE_CMOV */

  /* copy the stub beyond the end */
  i[2] = i[1];
  dl->length += 1;

  if (likely(pivot <= (ptrdiff_t)dl->sorted) &&
#if MDBX_HAVE_CMOV
      pivot_pgno < dp.pgno) {
#else
      (pivot <= 0 || dl->items[pivot].pgno < dp.pgno)) {
#endif /* MDBX_HAVE_CMOV */
    dl->sorted += 1;

    /* сдвигаем несортированный хвост */
    while (i >= dl->items + dl->sorted) {
#if !defined(__GNUC__) /* пытаемся избежать вызова memmove() */
      i[1] = *i;
#elif MDBX_WORDBITS == 64 && (defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128))
      STATIC_ASSERT(sizeof(dp) == sizeof(__uint128_t));
      ((__uint128_t *)i)[1] = *(volatile __uint128_t *)i;
#else
    i[1].ptr = i->ptr;
    i[1].pgno = i->pgno;
#if MDBX_DPL_CACHE_NPAGES
    i[1].npages = i->npages;
#endif /* MDBX_DPL_CACHE_NPAGES */
#endif
      --i;
    }
    /* ищем нужную позицию сдвигая отсортированные элементы */
    while (i->pgno > pgno) {
      cASSERT0(txn, i > dl->items);
      i[1] = *i;
      --i;
    }
    cASSERT0(txn, i->pgno < dp.pgno);
  }

  i[1] = dp;
  /* LY: здесь нет выхода за границу буфера, так как items[] выделяется с соответствующим запасом. */
  /* coverity[OVERRUN] */
  ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  ASSERT(dl->sorted <= dl->length);
  return MDBX_SUCCESS;
}

__cold bool txn_dpl_check(MDBX_txn *txn) {
  cASSERT0(txn, (txn->flags & txn_ro_flat) == 0);
  const dpl_t *const dl = txn->wr.dirtylist;
  if (!dl) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
    return true;
  }
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  ASSERT(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
  cASSERT0(txn,
           txn->wr.dirtyroom + dl->length == (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));

  if (!CHECKS2_ENABLED())
    return true;

  size_t loose = 0, pages = 0;
  for (size_t i = dl->length; i > 0; --i) {
    const page_t *const dp = dl->items[i].ptr;
    if (!dp)
      continue;

    cASSERT0(txn, dp->pgno == dl->items[i].pgno);
    if (unlikely(dp->pgno != dl->items[i].pgno))
      return false;

    if ((txn->flags & MDBX_WRITEMAP) == 0) {
      const uint32_t age = txn_dpl_age(txn, i);
      cASSERT0(txn, age < UINT32_MAX / 3);
      if (unlikely(age > UINT32_MAX / 3))
        return false;
    }

    cASSERT0(txn, dp->flags == P_LOOSE || is_modifable(txn, dp));
    if (dp->flags == P_LOOSE) {
      loose += 1;
    } else if (unlikely(!is_modifable(txn, dp)))
      return false;

    const unsigned num = dpl_npages(dl, i);
    pages += num;
    cASSERT0(txn, txn->geo.first_unallocated >= dp->pgno + num);
    if (unlikely(txn->geo.first_unallocated < dp->pgno + num))
      return false;

    if (i < dl->sorted) {
      cASSERT0(txn, dl->items[i + 1].pgno >= dp->pgno + num);
      if (unlikely(dl->items[i + 1].pgno < dp->pgno + num))
        return false;
    }

    const size_t rpa = pnl_search(txn->wr.repnl, dp->pgno, txn->geo.first_unallocated);
    cASSERT0(txn, rpa > pnl_size(txn->wr.repnl) || txn->wr.repnl[rpa] != dp->pgno);
    if (rpa <= pnl_size(txn->wr.repnl) && unlikely(txn->wr.repnl[rpa] == dp->pgno))
      return false;
    if (num > 1) {
      const size_t rpb = pnl_search(txn->wr.repnl, dp->pgno + num - 1, txn->geo.first_unallocated);
      cASSERT0(txn, rpa == rpb);
      if (unlikely(rpa != rpb))
        return false;
    }
  }

  cASSERT0(txn, loose == txn->wr.loose_count);
  if (unlikely(loose != txn->wr.loose_count))
    return false;

  cASSERT0(txn, pages == dl->pages_including_loose);
  if (unlikely(pages != dl->pages_including_loose))
    return false;

  for (size_t i = 1; i <= pnl_size(txn->wr.retired_pages); ++i) {
    const page_t *const dp = debug_txn_dpl_find(txn, txn->wr.retired_pages[i]);
    cASSERT0(txn, !dp);
    if (unlikely(dp))
      return false;
  }

  return true;
}

/*----------------------------------------------------------------------------*/

__noinline void txn_dpl_lru_reduce(MDBX_txn *txn) {
  VERBOSE("lru-reduce %u -> %u", txn->wr.dirtylru, txn->wr.dirtylru >> 1);
  cASSERT0(txn, (txn->flags & (txn_ro_both | MDBX_WRITEMAP)) == 0);
  do {
    txn->wr.dirtylru >>= 1;
    dpl_t *dl = txn->wr.dirtylist;
    for (size_t i = 1; i <= dl->length; ++i) {
      size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
      *ptr >>= 1;
    }
    txn = txn->parent;
  } while (txn);
}

void txn_dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
  if (pnl_size(pl) && txn->wr.dirtylist->length) {
    tASSERT1(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated << spilled));
    dpl_t *dl = txn_dpl_sort(txn);

    /* Scanning in ascend order */
    const intptr_t step = MDBX_PNL_ASCENDING ? 1 : -1;
    const intptr_t begin = MDBX_PNL_ASCENDING ? 1 : pnl_size(pl);
    const intptr_t end = MDBX_PNL_ASCENDING ? pnl_size(pl) + 1 : 0;
    cASSERT0(txn, pl[begin] <= pl[end - step]);

    size_t w, r = txn_dpl_search(txn, pl[begin] >> spilled);
    cASSERT0(txn, dl->sorted == dl->length);
    for (intptr_t i = begin; r <= dl->length;) { /* scan loop */
      ASSERT(i != end);
      cASSERT0(txn, !spilled || (pl[i] & 1) == 0);
      pgno_t pl_pgno = pl[i] >> spilled;
      pgno_t dp_pgno = dl->items[r].pgno;
      if (likely(dp_pgno != pl_pgno)) {
        const bool cmp = dp_pgno < pl_pgno;
        r += cmp;
        i += cmp ? 0 : step;
        if (likely(i != end))
          continue;
        return;
      }

      /* update loop */
      unsigned npages;
      w = r;
    remove_dl:
      npages = dpl_npages(dl, r);
      dl->pages_including_loose -= npages;
      if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP))
        page_shadow_release(txn->env, dl->items[r].ptr, npages);
      ++r;
    next_i:
      i += step;
      if (unlikely(i == end)) {
        while (r <= dl->length)
          dl->items[w++] = dl->items[r++];
      } else {
        while (r <= dl->length) {
          ASSERT(i != end);
          cASSERT0(txn, !spilled || (pl[i] & 1) == 0);
          pl_pgno = pl[i] >> spilled;
          dp_pgno = dl->items[r].pgno;
          if (dp_pgno < pl_pgno)
            dl->items[w++] = dl->items[r++];
          else if (dp_pgno > pl_pgno)
            goto next_i;
          else
            goto remove_dl;
        }
      }
      dl->sorted = dpl_setlen(dl, w - 1);
      txn->wr.dirtyroom += r - w;
      cASSERT0(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
                        (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
      return;
    }
  }
}

void txn_dpl_clear(MDBX_txn *txn) {
  cASSERT0(txn, (txn->flags & txn_ro_flat) == 0);
  dpl_t *const dl = txn->wr.dirtylist;
  if (dl) {
    if ((txn->flags & (txn_ro_nested | MDBX_WRITEMAP)) == 0) {
      MDBX_env *env = txn->env;
      for (size_t i = 1; i <= dl->length; i++)
        page_shadow_release(env, dl->items[i].ptr, dpl_npages(dl, i));
    }
    dpl_setlen(dl, 0);
  }
}

int dxb_msync(const MDBX_env *env, size_t length_pages, enum osal_syncmode_bits mode_bits) {
  if (MDBX_ENABLE_PGOP_STAT)
    env->lck->pgops.msync.weak += (MDBX_MMAP_NEEDS_JOLT || mode_bits > MDBX_SYNC_NONE);
  return osal_msync(&env->dxb_mmap, pgno_ceil2sp_bytes(env, length_pages), mode_bits);
}

int dxb_fsync(const MDBX_env *env, enum osal_syncmode_bits mode_bits) {
  if (MDBX_ENABLE_PGOP_STAT)
    env->lck->pgops.fsync.weak += (mode_bits > MDBX_SYNC_NONE);
  return osal_fsync(env->lazy_fd, mode_bits);
}

__cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, const mdbx_mode_t mode_bits) {
  memset(dest, 0, sizeof(meta_t));
  int rc = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  unaligned_poke_u64(4, dest->sign, DATASIGN_WEAK);
  rc = MDBX_CORRUPTED;

  /* Read twice all meta pages so we can find the latest one. */
  unsigned loop_limit = NUM_METAS * 2;
  /* We don't know the page size on first time. So, just guess it. */
  unsigned guess_pagesize = 0;
  for (unsigned loop_count = 0; loop_count < loop_limit; ++loop_count) {
    const unsigned meta_number = loop_count % NUM_METAS;
    const unsigned offset = (guess_pagesize             ? guess_pagesize
                             : (loop_count > NUM_METAS) ? env->ps
                                                        : globals.sys_pagesize) *
                            meta_number;

    char buffer[MDBX_MIN_PAGESIZE];
    unsigned retryleft = 42;
    while (1) {
      TRACE("reading meta[%d]: offset %u, bytes %u, retry-left %u", meta_number, offset, MDBX_MIN_PAGESIZE, retryleft);
      int err = osal_pread(env->lazy_fd, buffer, MDBX_MIN_PAGESIZE, offset);
      if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 && env->dxb_mmap.filesize == 0 &&
          mode_bits /* non-zero for DB creation */ != 0) {
        NOTICE("read meta: empty file (%d, %s)", err, mdbx_strerror(err));
        return err;
      }
#if defined(_WIN32) || defined(_WIN64)
      if (err == ERROR_LOCK_VIOLATION) {
        SleepEx(0, true);
        err = osal_pread(env->lazy_fd, buffer, MDBX_MIN_PAGESIZE, offset);
        if (err == ERROR_LOCK_VIOLATION && --retryleft) {
          WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err));
          continue;
        }
      }
#endif /* Windows */
      if (err != MDBX_SUCCESS) {
        ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err));
        return err;
      }

      char again[MDBX_MIN_PAGESIZE];
      err = osal_pread(env->lazy_fd, again, MDBX_MIN_PAGESIZE, offset);
#if defined(_WIN32) || defined(_WIN64)
      if (err == ERROR_LOCK_VIOLATION) {
        SleepEx(0, true);
        err = osal_pread(env->lazy_fd, again, MDBX_MIN_PAGESIZE, offset);
        if (err == ERROR_LOCK_VIOLATION && --retryleft) {
          WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err));
          continue;
        }
      }
#endif /* Windows */
      if (err != MDBX_SUCCESS) {
        ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err));
        return err;
      }

      if (memcmp(buffer, again, MDBX_MIN_PAGESIZE) == 0 || --retryleft == 0)
        break;

      VERBOSE("meta[%u] was updated, re-read it", meta_number);
    }

    if (!retryleft) {
      ERROR("meta[%u] is too volatile, skip it", meta_number);
      continue;
    }

    page_t *const page = (page_t *)buffer;
    meta_t *const meta = page_meta(page);
    rc = meta_validate(env, meta, page, meta_number, &guess_pagesize);
    if (rc != MDBX_SUCCESS)
      continue;

    bool latch;
    if (env->stuck_meta >= 0)
      latch = (meta_number == (unsigned)env->stuck_meta);
    else if (meta_bootid_match(meta))
      latch = meta_choice_recent(meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), dest->unsafe_txnid,
                                 SIGN_IS_STEADY(dest->unsafe_sign));
    else
      latch = meta_choice_steady(meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), dest->unsafe_txnid,
                                 SIGN_IS_STEADY(dest->unsafe_sign));
    if (latch) {
      *dest = *meta;
      if (!lck_exclusive && !meta_is_steady(dest))
        loop_limit += 1; /* LY: should re-read to hush race with update */
      VERBOSE("latch meta[%u]", meta_number);
    }
  }

  if (dest->pagesize == 0 ||
      (env->stuck_meta < 0 && !(meta_is_steady(dest) || meta_weak_acceptable(env, dest, lck_exclusive)))) {
    ERROR("%s", "no usable meta-pages, database is corrupted");
    if (!MDBX_IS_ERROR(rc)) {
      /* TODO: try to restore the database by fully checking b-tree structure
       * for the each meta page, if the corresponding option was given */
      return MDBX_CORRUPTED;
    }
    return rc;
  }

  return MDBX_SUCCESS;
}

__cold int dxb_resize(MDBX_env *const env, const pgno_t allocated_pgno, const pgno_t size_pgno, pgno_t limit_pgno,
                      const enum resize_mode mode) {
  /* Acquire guard to avoid collision between read and write txns
   * around geo_in_bytes and dxb_mmap */
#if defined(_WIN32) || defined(_WIN64)
  imports.srwl_AcquireExclusive(&env->remap_lock);
  int rc = MDBX_SUCCESS;
  mdbx_handle_array_t *suspended = nullptr;
  mdbx_handle_array_t array_onstack;
#else
  int rc = osal_fastmutex_acquire(&env->remap_lock);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;
#endif

  const size_t prev_size = env->dxb_mmap.current;
  const size_t prev_limit = env->dxb_mmap.limit;
  const pgno_t prev_limit_pgno = bytes2pgno(env, prev_limit);
  eASSERT0(env, limit_pgno >= size_pgno);
  eASSERT0(env, size_pgno >= allocated_pgno);
  if (mode < explicit_resize && size_pgno <= prev_limit_pgno) {
    /* The actual mapsize may be less since the geo.upper may be changed
     * by other process. Avoids remapping until it necessary. */
    limit_pgno = prev_limit_pgno;
  }
  const size_t limit_bytes = pgno_ceil2os_bytes(env, limit_pgno);
  const size_t size_bytes = pgno_ceil2os_bytes(env, size_pgno);
  const void *const prev_map = env->dxb_mmap.base;

  VERBOSE("resize(env-flags 0x%x, mode %d) datafile/mapping: "
          "present %" PRIuPTR " -> %" PRIuPTR ", "
          "limit %" PRIuPTR " -> %" PRIuPTR,
          env->flags, mode, prev_size, size_bytes, prev_limit, limit_bytes);

  eASSERT0(env, limit_bytes >= size_bytes);
  eASSERT0(env, bytes2pgno(env, size_bytes) >= size_pgno);
  eASSERT0(env, bytes2pgno(env, limit_bytes) >= limit_pgno);

  unsigned mresize_flags = env->flags & (MDBX_RDONLY | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC);
  if (mode >= impilict_shrink)
    mresize_flags |= txn_shrink_allowed;

  if (limit_bytes == env->dxb_mmap.limit && size_bytes == env->dxb_mmap.current && size_bytes == env->dxb_mmap.filesize)
    goto bailout;

  /* При использовании MDBX_NOSTICKYTHREADS с транзакциями могут работать любые
   * потоки и у нас нет информации о том, какие именно. Поэтому нет возможности
   * выполнить remap-действия требующие приостановки работающих с БД потоков. */
  if ((env->flags & MDBX_NOSTICKYTHREADS) == 0) {
#if defined(_WIN32) || defined(_WIN64)
    if ((size_bytes < env->dxb_mmap.current && mode > implicit_grow) || limit_bytes != env->dxb_mmap.limit) {
      /* 1) Windows allows only extending a read-write section, but not a
       *    corresponding mapped view. Therefore in other cases we must suspend
       *    the local threads for safe remap.
       * 2) At least on Windows 10 1803 the entire mapped section is unavailable
       *    for short time during NtExtendSection() or VirtualAlloc() execution.
       * 3) Under Wine runtime environment on Linux a section extending is not
       *    supported.
       *
       * THEREFORE LOCAL THREADS SUSPENDING IS ALWAYS REQUIRED! */
      array_onstack.limit = ARRAY_LENGTH(array_onstack.handles);
      array_onstack.count = 0;
      suspended = &array_onstack;
      rc = osal_suspend_threads_before_remap(env, &suspended);
      if (rc != MDBX_SUCCESS) {
        ERROR("failed suspend-for-remap: errcode %d", rc);
        goto bailout;
      }
      mresize_flags |=
          (mode < explicit_resize) ? MDBX_MRESIZE_MAY_UNMAP : MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE;
    }
#else  /* Windows */
    lck_t *const lck = env->lck_mmap.lck;
    if (mode == explicit_resize && limit_bytes != env->dxb_mmap.limit) {
      mresize_flags |= MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE;
      if (lck) {
        int err = lck_rdt_lock(env) /* lock readers table until remap done */;
        if (unlikely(MDBX_IS_ERROR(err))) {
          rc = err;
          goto bailout;
        }

        /* looking for readers from this process */
        const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
        eASSERT0(env, mode == explicit_resize);
        for (size_t i = 0; i < snap_nreaders; ++i) {
          if (lck->rdt[i].pid.weak == (size_t)env->registered_reader_pid &&
              lck->rdt[i].tid.weak != osal_thread_self() && lck->rdt[i].tid.weak < MDBX_TID_TXN_OUSTED) {
            /* the base address of the mapping can't be changed since
             * the other reader thread from this process exists. */
            lck_rdt_unlock(env);
            mresize_flags &= ~(MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE);
            break;
          }
        }
      }
    }
#endif /* ! Windows */
  }

  const pgno_t aligned_munlock_pgno =
      (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) ? 0 : bytes2pgno(env, size_bytes);
  if (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) {
    env_clear_incore_cache(env);
    if ((env->flags & MDBX_WRITEMAP) && env->lck->unsynced_pages.weak) {
      rc = dxb_msync(env, allocated_pgno, MDBX_SYNC_KICK);
      if (unlikely(rc != MDBX_SUCCESS))
        goto bailout;
    }
  }
  munlock_after(env, aligned_munlock_pgno, size_bytes);

  if (size_bytes < prev_size && mode > implicit_grow) {
    NOTICE("resize-MADV_%s %u..%u", (env->flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED", size_pgno,
           bytes2pgno(env, prev_size));
    const uint32_t munlocks_before = atomic_load32(&env->lck->mlcnt[1], mo_Relaxed);
    rc = MDBX_RESULT_TRUE;
#if defined(MADV_REMOVE)
    if (env->flags & MDBX_WRITEMAP)
      rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, MADV_REMOVE)
               ? ignore_enosys_and_eagain(errno)
               : MDBX_SUCCESS;
#endif /* MADV_REMOVE */
#if defined(MADV_DONTNEED)
    if (rc == MDBX_RESULT_TRUE)
      rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, MADV_DONTNEED)
               ? ignore_enosys_and_eagain(errno)
               : MDBX_SUCCESS;
#elif defined(POSIX_MADV_DONTNEED)
    if (rc == MDBX_RESULT_TRUE)
      rc = ignore_enosys(
          posix_madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, POSIX_MADV_DONTNEED));
#elif defined(POSIX_FADV_DONTNEED)
    if (rc == MDBX_RESULT_TRUE)
      rc = ignore_enosys(posix_fadvise(env->lazy_fd, size_bytes, prev_size - size_bytes, POSIX_FADV_DONTNEED));
#endif /* MADV_DONTNEED */
    if (unlikely(MDBX_IS_ERROR(rc))) {
      const uint32_t mlocks_after = atomic_load32(&env->lck->mlcnt[0], mo_Relaxed);
      if (rc == MDBX_EINVAL) {
        const int severity = (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN;
        if (LOG_ENABLED(severity))
          debug_log(severity, __func__, __LINE__,
                    "%s-madvise: ignore EINVAL (%d) since some pages maybe "
                    "locked (%u/%u mlcnt-processes)",
                    "resize", rc, mlocks_after, munlocks_before);
      } else {
        ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", "mresize", "DONTNEED", size_bytes,
              prev_size - size_bytes, mlocks_after, munlocks_before, rc);
        goto bailout;
      }
    } else
      env->lck->discarded_tail.weak = size_pgno;
  }

  rc = osal_mresize(mresize_flags, &env->dxb_mmap, size_bytes, limit_bytes);
  eASSERT0(env, env->dxb_mmap.limit >= env->dxb_mmap.current);

  if (rc == MDBX_SUCCESS) {
    eASSERT0(env, limit_bytes == env->dxb_mmap.limit);
    eASSERT0(env, size_bytes <= env->dxb_mmap.filesize);
    if (mode == explicit_resize)
      eASSERT0(env, size_bytes == env->dxb_mmap.current);
    else
      eASSERT0(env, size_bytes <= env->dxb_mmap.current);
    env->lck->discarded_tail.weak = size_pgno;
    const bool readahead =
        !(env->flags & MDBX_NORDAHEAD) && mdbx_is_readahead_reasonable(size_bytes, -(intptr_t)prev_size);
    const bool force = limit_bytes != prev_limit || env->dxb_mmap.base != prev_map
#if defined(_WIN32) || defined(_WIN64)
                       || prev_size > size_bytes
#endif /* Windows */
        ;
    rc = dxb_set_readahead(env, size_pgno, readahead, force);
  }

bailout:
  if (rc == MDBX_SUCCESS) {
    eASSERT0(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
    eASSERT0(env, limit_bytes == env->dxb_mmap.limit);
    eASSERT0(env, size_bytes <= env->dxb_mmap.filesize);
    if (mode == explicit_resize)
      eASSERT0(env, size_bytes == env->dxb_mmap.current);
    else
      eASSERT0(env, size_bytes <= env->dxb_mmap.current);
    /* update env-geo to avoid influences */
    env->geo_in_bytes.now = env->dxb_mmap.current;
    env->geo_in_bytes.upper = env->dxb_mmap.limit;
    env_options_adjust_defaults(env);
#ifdef ENABLE_MEMCHECK
    if (prev_limit != env->dxb_mmap.limit || prev_map != env->dxb_mmap.base) {
      VALGRIND_DISCARD(env->valgrind_handle);
      env->valgrind_handle = 0;
      if (env->dxb_mmap.limit)
        env->valgrind_handle = VALGRIND_CREATE_BLOCK(env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx");
    }
#endif /* ENABLE_MEMCHECK */
  } else {
    if (rc != MDBX_UNABLE_EXTEND_MAPSIZE && rc != MDBX_EPERM) {
      ERROR("failed resize datafile/mapping: "
            "present %" PRIuPTR " -> %" PRIuPTR ", "
            "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d",
            prev_size, size_bytes, prev_limit, limit_bytes, rc);
    } else {
      WARNING("unable resize datafile/mapping: "
              "present %" PRIuPTR " -> %" PRIuPTR ", "
              "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d",
              prev_size, size_bytes, prev_limit, limit_bytes, rc);
      eASSERT0(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
    }
    if (!env->dxb_mmap.base) {
      env->flags |= ENV_FATAL_ERROR;
      if (env->txn)
        env->txn->flags |= MDBX_TXN_ERROR;
      rc = MDBX_PANIC;
    }
  }

#if defined(_WIN32) || defined(_WIN64)
  int err = MDBX_SUCCESS;
  imports.srwl_ReleaseExclusive(&env->remap_lock);
  if (suspended) {
    err = osal_resume_threads_after_remap(suspended);
    if (suspended != &array_onstack)
      osal_free(suspended);
  }
#else
  if (env->lck_mmap.lck && (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) != 0)
    lck_rdt_unlock(env);
  int err = osal_fastmutex_release(&env->remap_lock);
#endif /* Windows */
  if (err != MDBX_SUCCESS) {
    FATAL("failed resume-after-remap: errcode %d", err);
    return MDBX_PANIC;
  }
  return rc;
}
#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) {
#if !defined(__SANITIZE_ADDRESS__)
  if (!RUNNING_ON_VALGRIND)
    return;
#endif
  if (txn) { /* transaction start */
    if (env->poison_edge < txn->geo.first_unallocated)
      env->poison_edge = txn->geo.first_unallocated;
    VALGRIND_MAKE_MEM_DEFINED(env->dxb_mmap.base, pgno2bytes(env, txn->geo.first_unallocated));
    MDBX_ASAN_UNPOISON_MEMORY_REGION(env->dxb_mmap.base, pgno2bytes(env, txn->geo.first_unallocated));
    /* don't touch more, it should be already poisoned */
  } else { /* transaction end */
    bool should_unlock = false;
    pgno_t last = MAX_PAGENO + 1;
    if (env->pid != osal_getpid()) {
      /* resurrect after fork */
      return;
    } else if (env_owned_wrtxn(env)) {
      /* inside write-txn */
      last = meta_recent(env, &env->basal_txn->wr.troika).ptr_v->geometry.first_unallocated;
    } else if (env->flags & MDBX_RDONLY) {
      /* read-only mode, no write-txn, no wlock mutex */
      last = NUM_METAS;
    } else if (lck_txn_lock(env, true) == MDBX_SUCCESS) {
      /* no write-txn */
      last = NUM_METAS;
      should_unlock = true;
    } else {
      /* write txn is running, therefore shouldn't poison any memory range */
      return;
    }

    last = mvcc_largest_this(env, last);
    const pgno_t edge = env->poison_edge;
    if (edge > last) {
      eASSERT0(env, last >= NUM_METAS);
      env->poison_edge = last;
      VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), pgno2bytes(env, edge - last));
      MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), pgno2bytes(env, edge - last));
    }
    if (should_unlock)
      lck_txn_unlock(env);
  }
}
#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */

/* Turn on/off readahead. It's harmful when the DB is larger than RAM. */
__cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, const bool enable, const bool force_whole) {
  eASSERT0(env, edge >= NUM_METAS && edge <= MAX_PAGENO + 1);
  eASSERT0(env, (enable & 1) == (enable != 0));
  const bool toggle = force_whole || ((enable ^ env->lck->readahead_anchor) & 1) || !env->lck->readahead_anchor;
  const pgno_t prev_edge = env->lck->readahead_anchor >> 1;
  const size_t limit = env->dxb_mmap.limit;
  size_t offset = toggle ? 0 : pgno_ceil2os_bytes(env, (prev_edge < edge) ? prev_edge : edge);
  offset = (offset < limit) ? offset : limit;

  size_t length = pgno_ceil2os_bytes(env, (prev_edge < edge) ? edge : prev_edge);
  length = (length < limit) ? length : limit;
  length -= offset;

  eASSERT0(env, 0 <= (intptr_t)length);
  if (length == 0)
    return MDBX_SUCCESS;

  NOTICE("readahead %s %u..%u", enable ? "ON" : "OFF", bytes2pgno(env, offset), bytes2pgno(env, offset + length));

#if defined(F_RDAHEAD)
  if (toggle && unlikely(fcntl(env->lazy_fd, F_RDAHEAD, enable) == -1))
    return errno;
#endif /* F_RDAHEAD */

  int err;
  void *const ptr = ptr_disp(env->dxb_mmap.base, offset);
  if (enable) {
#if defined(MADV_NORMAL)
    err = madvise(ptr, length, MADV_NORMAL) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(POSIX_MADV_NORMAL)
    err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_NORMAL));
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(POSIX_FADV_NORMAL) && defined(POSIX_FADV_WILLNEED)
    err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_NORMAL));
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(_WIN32) || defined(_WIN64)
    /* no madvise on Windows */
#else
#warning "FIXME"
#endif
    if (toggle) {
      /* NOTE: Seems there is a bug in the Mach/Darwin/OSX kernel,
       * because MADV_WILLNEED with offset != 0 may cause SIGBUS
       * on following access to the hinted region.
       * 19.6.0 Darwin Kernel Version 19.6.0: Tue Jan 12 22:13:05 PST 2021;
       * root:xnu-6153.141.16~1/RELEASE_X86_64 x86_64 */
#if defined(F_RDADVISE)
      struct radvisory hint;
      hint.ra_offset = offset;
      hint.ra_count = unlikely(length > INT_MAX && sizeof(length) > sizeof(hint.ra_count)) ? INT_MAX : (int)length;
      (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl(env->lazy_fd, F_RDADVISE, &hint);
#elif defined(MADV_WILLNEED)
      err = madvise(ptr, length, MADV_WILLNEED) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
      if (unlikely(MDBX_IS_ERROR(err)))
        return err;
#elif defined(POSIX_MADV_WILLNEED)
      err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_WILLNEED));
      if (unlikely(MDBX_IS_ERROR(err)))
        return err;
#elif defined(_WIN32) || defined(_WIN64)
      if (imports.PrefetchVirtualMemory) {
        WIN32_MEMORY_RANGE_ENTRY hint;
        hint.VirtualAddress = ptr;
        hint.NumberOfBytes = length;
        (void)imports.PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0);
      }
#elif defined(POSIX_FADV_WILLNEED)
      err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_WILLNEED));
      if (unlikely(MDBX_IS_ERROR(err)))
        return err;
#else
#warning "FIXME"
#endif
    }
  } else {
    env_clear_incore_cache(env);
#if defined(MADV_RANDOM)
    err = madvise(ptr, length, MADV_RANDOM) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(POSIX_MADV_RANDOM)
    err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_RANDOM));
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(POSIX_FADV_RANDOM)
    err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_RANDOM));
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(_WIN32) || defined(_WIN64)
    /* no madvise on Windows */
#else
#warning "FIXME"
#endif /* MADV_RANDOM */
  }

  env->lck->readahead_anchor = (enable & 1) + (edge << 1);
  err = MDBX_SUCCESS;
  return err;
}

__cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bits) {
  meta_t header;
  eASSERT0(env, !(env->flags & ENV_ACTIVE));
  int rc = MDBX_RESULT_FALSE;
  int err = dxb_read_header(env, &header, lck_rc, mode_bits);
  if (unlikely(err != MDBX_SUCCESS)) {
    if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA || (env->flags & MDBX_RDONLY) != 0 ||
        /* recovery mode */ env->stuck_meta >= 0)
      return err;

    DEBUG("%s", "create new database");
    rc = /* new database */ MDBX_RESULT_TRUE;

    if (!env->geo_in_bytes.now) {
      /* set defaults if not configured */
      err = mdbx_env_set_geometry(env, 0, -1, -1, -1, -1, -1);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }

    err = env_page_auxbuffer(env);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    header = *meta_init_triplet(env, env->page_auxbuf);
    err = osal_pwrite(env->lazy_fd, env->page_auxbuf, env->ps * (size_t)NUM_METAS, 0);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    err = osal_fsetsize(env->lazy_fd, env->dxb_mmap.filesize = env->dxb_mmap.current = env->geo_in_bytes.now);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

#if MDBX_CHECKING > 0 || MDBX_DEBUG > 0
    err = dxb_read_header(env, &header, lck_rc, mode_bits);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
#endif /* MDBX_CHECKING > 0 || MDBX_DEBUG > 0 */
  }

  VERBOSE("header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
          " +%u -%u, txn_id %" PRIaTXN ", %s",
          header.trees.main.root, header.trees.gc.root, header.geometry.lower, header.geometry.first_unallocated,
          header.geometry.now, header.geometry.upper, pv2pages(header.geometry.grow_pv),
          pv2pages(header.geometry.shrink_pv), unaligned_peek_u64(4, header.txnid_a), durable_caption(&header));

  if (unlikely((header.trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY)) {
    ERROR("unexpected/invalid db-flags 0x%x for %s", header.trees.gc.flags, "GC/FreeDB");
    return MDBX_INCOMPATIBLE;
  }
  env->dbs_flags[FREE_DBI] = DB_VALID | MDBX_INTEGERKEY;
  env->kvs[FREE_DBI].clc.k.cmp = cmp_uint_align4; /* aligned MDBX_INTEGERKEY */
  env->kvs[FREE_DBI].clc.k.lmax = env->kvs[FREE_DBI].clc.k.lmin = 8;
  env->kvs[FREE_DBI].clc.v.cmp = cmp_lenfast;
  env->kvs[FREE_DBI].clc.v.lmin = 4;
  env->kvs[FREE_DBI].clc.v.lmax = mdbx_env_get_maxvalsize_ex(env, MDBX_INTEGERKEY);
  clc_reset_methods(&env->kvs[FREE_DBI].clc.k);
  clc_reset_methods(&env->kvs[FREE_DBI].clc.v);

  if (env->ps != header.pagesize)
    env_setup_pagesize(env, header.pagesize);
  if ((env->flags & MDBX_RDONLY) == 0) {
    err = env_page_auxbuffer(env);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }

  size_t expected_filesize = 0;
  const size_t allocated_bytes = pgno2bytes(env, header.geometry.first_unallocated);
  const size_t allocated_aligned2os_bytes = ceil_powerof2(allocated_bytes, globals.sys_allocation_granularity);
  if ((env->flags & MDBX_RDONLY)    /* readonly */
      || lck_rc != MDBX_RESULT_TRUE /* not exclusive */
      || /* recovery mode */ env->stuck_meta >= 0) {
    /* use present params from db */
    const size_t pagesize = header.pagesize;
    err = mdbx_env_set_geometry(env, header.geometry.lower * pagesize, header.geometry.now * pagesize,
                                header.geometry.upper * pagesize, pv2pages(header.geometry.grow_pv) * pagesize,
                                pv2pages(header.geometry.shrink_pv) * pagesize, header.pagesize);
    if (unlikely(err != MDBX_SUCCESS)) {
      ERROR("%s: err %d", "could not apply geometry from db", err);
      return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err;
    }
  } else if (env->geo_in_bytes.now) {
    /* silently growth to last used page */
    if (env->geo_in_bytes.now < allocated_aligned2os_bytes)
      env->geo_in_bytes.now = allocated_aligned2os_bytes;
    if (env->geo_in_bytes.upper < allocated_aligned2os_bytes)
      env->geo_in_bytes.upper = allocated_aligned2os_bytes;

    /* apply preconfigured params, but only if substantial changes:
     *  - upper or lower limit changes
     *  - shrink threshold or growth step
     * But ignore change just a 'now/current' size. */
    if (bytes_ceil2os_bytes(env, env->geo_in_bytes.upper) != pgno2bytes(env, header.geometry.upper) ||
        bytes_ceil2os_bytes(env, env->geo_in_bytes.lower) != pgno2bytes(env, header.geometry.lower) ||
        bytes_ceil2os_bytes(env, env->geo_in_bytes.shrink) != pgno2bytes(env, pv2pages(header.geometry.shrink_pv)) ||
        bytes_ceil2os_bytes(env, env->geo_in_bytes.grow) != pgno2bytes(env, pv2pages(header.geometry.grow_pv))) {

      if (env->geo_in_bytes.shrink && env->geo_in_bytes.now > allocated_bytes)
        /* pre-shrink if enabled */
        env->geo_in_bytes.now = allocated_bytes + env->geo_in_bytes.shrink - allocated_bytes % env->geo_in_bytes.shrink;

      /* сейчас БД еще не открыта, поэтому этот вызов не изменит геометрию, но проверит и скорректирует параметры
       * с учетом реального размера страницы. */
      err = mdbx_env_set_geometry(env, env->geo_in_bytes.lower, env->geo_in_bytes.now, env->geo_in_bytes.upper,
                                  env->geo_in_bytes.grow, env->geo_in_bytes.shrink, header.pagesize);
      if (unlikely(err != MDBX_SUCCESS)) {
        ERROR("%s: err %d", "could not apply preconfigured db-geometry", err);
        return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err;
      }

      /* altering fields to match geometry given from user */
      expected_filesize = pgno_ceil2os_bytes(env, header.geometry.now);
      header.geometry.now = bytes_ceil2os_pgno(env, env->geo_in_bytes.now);
      header.geometry.lower = bytes_ceil2os_pgno(env, env->geo_in_bytes.lower);
      header.geometry.upper = bytes_ceil2os_pgno(env, env->geo_in_bytes.upper);
      header.geometry.grow_pv = pages2pv(bytes_ceil2os_pgno(env, env->geo_in_bytes.grow));
      header.geometry.shrink_pv = pages2pv(bytes_ceil2os_pgno(env, env->geo_in_bytes.shrink));

      VERBOSE("amending: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
              " +%u -%u, txn_id %" PRIaTXN ", %s",
              header.trees.main.root, header.trees.gc.root, header.geometry.lower, header.geometry.first_unallocated,
              header.geometry.now, header.geometry.upper, pv2pages(header.geometry.grow_pv),
              pv2pages(header.geometry.shrink_pv), unaligned_peek_u64(4, header.txnid_a), durable_caption(&header));
    } else {
      /* fetch back 'now/current' size, since it was ignored during comparison and may differ. */
      env->geo_in_bytes.now = pgno_ceil2os_bytes(env, header.geometry.now);
    }
    ENSURE_OBJ(env, header.geometry.now >= header.geometry.first_unallocated);
  } else {
    /* geo-params are not pre-configured by user, get current values from the meta. */
    env->geo_in_bytes.now = pgno_ceil2os_bytes(env, header.geometry.now);
    env->geo_in_bytes.lower = pgno_ceil2os_bytes(env, header.geometry.lower);
    env->geo_in_bytes.upper = pgno_ceil2os_bytes(env, header.geometry.upper);
    env->geo_in_bytes.grow = pgno_ceil2os_bytes(env, pv2pages(header.geometry.grow_pv));
    env->geo_in_bytes.shrink = pgno_ceil2os_bytes(env, pv2pages(header.geometry.shrink_pv));
  }

  ENSURE_OBJ(env, env->geo_in_bytes.now >= allocated_bytes);
  if (!expected_filesize)
    expected_filesize = env->geo_in_bytes.now;
  const uint64_t filesize_before = env->dxb_mmap.filesize;
  if (unlikely(filesize_before != env->geo_in_bytes.now)) {
    if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) {
      VERBOSE("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO "p, have %" PRIu64 "b/%" PRIu64
              "p), assume other process working",
              env->geo_in_bytes.now, bytes2pgno(env, env->geo_in_bytes.now), filesize_before,
              filesize_before >> env->ps2ln);
    } else {
      if (filesize_before != expected_filesize)
        WARNING("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO "p, have %" PRIu64 "b/%" PRIu64 "p)",
                expected_filesize, bytes2pgno(env, expected_filesize), filesize_before, filesize_before >> env->ps2ln);
      if (filesize_before < allocated_bytes) {
        ERROR("last-page beyond end-of-file (last %" PRIaPGNO ", have %" PRIaPGNO ")",
              header.geometry.first_unallocated, bytes2pgno(env, (size_t)filesize_before));
        return MDBX_CORRUPTED;
      }

      if (env->flags & MDBX_RDONLY) {
        if (filesize_before & (globals.sys_pagesize - 1)) {
          ERROR("filesize should be rounded-up to system page size %u", globals.sys_pagesize);
          return MDBX_WANNA_RECOVERY;
        }
        WARNING("%s", "ignore filesize mismatch in readonly-mode");
      } else {
        VERBOSE("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO " pages", env->geo_in_bytes.now,
                bytes2pgno(env, env->geo_in_bytes.now));
      }
    }
  }

  VERBOSE("current boot-id %" PRIx64 "-%" PRIx64 " (%savailable)", globals.bootid.x, globals.bootid.y,
          (globals.bootid.x | globals.bootid.y) ? "" : "not-");

  /* calculate readahead hint before mmap with zero redundant pages */
  const bool readahead =
      !(env->flags & MDBX_NORDAHEAD) && mdbx_is_readahead_reasonable(allocated_bytes, 0) == MDBX_RESULT_TRUE;

  err = osal_mmap(env->flags, &env->dxb_mmap, env->geo_in_bytes.now, env->geo_in_bytes.upper,
                  (lck_rc && env->stuck_meta < 0) ? MMAP_OPTION_SETLENGTH : 0, env->pathname.dxb);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

#if defined(MADV_DONTDUMP)
  err =
      madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
  if (unlikely(MDBX_IS_ERROR(err)))
    return err;
#endif /* MADV_DONTDUMP */
#if defined(MADV_DODUMP)
  if (globals.runtime_flags & MDBX_DBG_DUMP) {
    const size_t meta_length_aligned2os = pgno_ceil2os_bytes(env, NUM_METAS);
    err = madvise(env->dxb_mmap.base, meta_length_aligned2os, MADV_DODUMP) ? ignore_enosys_and_eagain(errno)
                                                                           : MDBX_SUCCESS;
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
  }
#endif /* MADV_DODUMP */

#ifdef ENABLE_MEMCHECK
  env->valgrind_handle = VALGRIND_CREATE_BLOCK(env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx");
#endif /* ENABLE_MEMCHECK */

  eASSERT0(env, allocated_bytes >= pgno2bytes(env, NUM_METAS) && allocated_bytes <= env->dxb_mmap.limit);
#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
  if (env->dxb_mmap.filesize > allocated_bytes && env->dxb_mmap.filesize < env->dxb_mmap.limit) {
    VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, allocated_bytes), env->dxb_mmap.filesize - allocated_bytes);
    MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, allocated_bytes),
                                   env->dxb_mmap.filesize - allocated_bytes);
  }
  env->poison_edge =
      bytes2pgno(env, (env->dxb_mmap.filesize < env->dxb_mmap.limit) ? env->dxb_mmap.filesize : env->dxb_mmap.limit);
#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */

  troika_t troika = meta_tap(env);
#if MDBX_DEBUG > 0
  meta_troika_dump(env, &troika);
#endif
  //-------------------------------- validate/rollback head & steady meta-pages
  if (unlikely(env->stuck_meta >= 0)) {
    /* recovery mode */
    meta_t clone;
    meta_t const *const target = METAPAGE(env, env->stuck_meta);
    err = meta_validate_copy(env, target, &clone);
    if (unlikely(err != MDBX_SUCCESS)) {
      ERROR("target meta[%u] is corrupted", bytes2pgno(env, ptr_dist(payload2page(target), env->dxb_mmap.base)));
      meta_troika_dump(env, &troika);
      return MDBX_CORRUPTED;
    }
  } else /* not recovery mode */
    while (1) {
      const unsigned meta_clash_mask = meta_eq_mask(&troika);
      if (unlikely(meta_clash_mask)) {
        ERROR("meta-pages are clashed: mask 0x%d", meta_clash_mask);
        meta_troika_dump(env, &troika);
        return MDBX_CORRUPTED;
      }

      if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) {
        /* non-exclusive mode,
         * meta-pages should be validated by a first process opened the DB */
        if (troika.recent == troika.prefer_steady)
          break;

        if (!env->lck_mmap.lck) {
          /* LY: without-lck (read-only) mode, so it is impossible that other
           * process made weak checkpoint. */
          ERROR("%s", "without-lck, unable recovery/rollback");
          meta_troika_dump(env, &troika);
          return MDBX_WANNA_RECOVERY;
        }

        /* LY: assume just have a collision with other running process,
         *     or someone make a weak checkpoint */
        VERBOSE("%s", "assume collision or online weak checkpoint");
        break;
      }
      eASSERT0(env, lck_rc == MDBX_RESULT_TRUE);
      /* exclusive mode */

      const meta_ptr_t recent = meta_recent(env, &troika);
      const meta_ptr_t prefer_steady = meta_prefer_steady(env, &troika);
      meta_t clone;
      if (prefer_steady.is_steady) {
        err = meta_validate_copy(env, prefer_steady.ptr_c, &clone);
        if (unlikely(err != MDBX_SUCCESS)) {
          ERROR("meta[%u] with %s txnid %" PRIaTXN " is corrupted, %s needed",
                bytes2pgno(env, ptr_dist(prefer_steady.ptr_c, env->dxb_mmap.base)), "steady", prefer_steady.txnid,
                "manual recovery");
          meta_troika_dump(env, &troika);
          return MDBX_CORRUPTED;
        }
        if (prefer_steady.ptr_c == recent.ptr_c)
          break;
      }

      const pgno_t pgno = bytes2pgno(env, ptr_dist(recent.ptr_c, env->dxb_mmap.base));
      const bool last_valid = meta_validate_copy(env, recent.ptr_c, &clone) == MDBX_SUCCESS;
      eASSERT0(env, !prefer_steady.is_steady || recent.txnid != prefer_steady.txnid);
      if (unlikely(!last_valid)) {
        if (unlikely(!prefer_steady.is_steady)) {
          ERROR("%s for open or automatic rollback, %s", "there are no suitable meta-pages",
                "manual recovery is required");
          meta_troika_dump(env, &troika);
          return MDBX_CORRUPTED;
        }
        WARNING("meta[%u] with last txnid %" PRIaTXN " is corrupted, rollback needed", pgno, recent.txnid);
        meta_troika_dump(env, &troika);
        goto purge_meta_head;
      }

      if (meta_bootid_match(recent.ptr_c)) {
        if (env->flags & MDBX_RDONLY) {
          ERROR("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
                "rollback NOT needed, steady-sync NEEDED%s",
                "opening after an unclean shutdown", globals.bootid.x, globals.bootid.y,
                ", but unable in read-only mode");
          meta_troika_dump(env, &troika);
          return MDBX_WANNA_RECOVERY;
        }
        WARNING("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
                "rollback NOT needed, steady-sync NEEDED%s",
                "opening after an unclean shutdown", globals.bootid.x, globals.bootid.y, "");
        header = clone;
        env->lck->unsynced_pages.weak = header.geometry.first_unallocated;
        if (!env->lck->eoos_timestamp.weak)
          env->lck->eoos_timestamp.weak = osal_monotime();
        break;
      }
      if (unlikely(!prefer_steady.is_steady)) {
        ERROR("%s, but %s for automatic rollback: %s", "opening after an unclean shutdown",
              "there are no suitable meta-pages", "manual recovery is required");
        meta_troika_dump(env, &troika);
        return MDBX_CORRUPTED;
      }

    purge_meta_head:
      if (env->flags & MDBX_RDONLY) {
        ERROR("%s and rollback needed: (from head %" PRIaTXN " to steady %" PRIaTXN ")%s",
              "opening after an unclean shutdown", recent.txnid, prefer_steady.txnid, ", but unable in read-only mode");
        meta_troika_dump(env, &troika);
        return MDBX_WANNA_RECOVERY;
      }
      NOTICE("%s and doing automatic rollback: "
             "purge%s meta[%u] with%s txnid %" PRIaTXN,
             "opening after an unclean shutdown", last_valid ? "" : " invalid", pgno, last_valid ? " weak" : "",
             recent.txnid);
      meta_troika_dump(env, &troika);
      ENSURE_OBJ(env, prefer_steady.is_steady);
      err = meta_override(env, pgno, 0, last_valid ? recent.ptr_c : prefer_steady.ptr_c);
      if (err) {
        ERROR("rollback: overwrite meta[%u] with txnid %" PRIaTXN ", error %d", pgno, recent.txnid, err);
        return err;
      }
      troika = meta_tap(env);
      ENSURE_OBJ(env, 0 == meta_txnid(recent.ptr_v));
      ENSURE_OBJ(env, 0 == meta_eq_mask(&troika));
    }

  if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) {
    //-------------------------------------------------- shrink DB & update geo
    /* re-check size after mmap */
    if (floor_powerof2(env->dxb_mmap.current, globals.sys_pagesize) < allocated_bytes) {
      ERROR("unacceptable/unexpected datafile size %" PRIuPTR, env->dxb_mmap.current);
      return MDBX_PROBLEM;
    }
    if (env->dxb_mmap.current != env->geo_in_bytes.now) {
      header.geometry.now = bytes2pgno(env, env->geo_in_bytes.now);
      NOTICE("need update meta-geo to filesize %" PRIuPTR " bytes, aligned %" PRIaPGNO " pages", env->geo_in_bytes.now,
             header.geometry.now);
    }

    const meta_ptr_t recent = meta_recent(env, &troika);
    if (/* не учитываем различия в geo.first_unallocated */
        header.geometry.grow_pv != recent.ptr_c->geometry.grow_pv ||
        header.geometry.shrink_pv != recent.ptr_c->geometry.shrink_pv ||
        header.geometry.lower != recent.ptr_c->geometry.lower ||
        header.geometry.upper != recent.ptr_c->geometry.upper || header.geometry.now != recent.ptr_c->geometry.now) {
      if ((env->flags & MDBX_RDONLY) != 0 ||
          /* recovery mode */ env->stuck_meta >= 0) {
        WARNING("skipped update meta.geo in %s mode: from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO
                "/s%u-g%u, to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u",
                (env->stuck_meta < 0) ? "read-only" : "recovery", recent.ptr_c->geometry.lower,
                recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper, pv2pages(recent.ptr_c->geometry.shrink_pv),
                pv2pages(recent.ptr_c->geometry.grow_pv), header.geometry.lower, header.geometry.now,
                header.geometry.upper, pv2pages(header.geometry.shrink_pv), pv2pages(header.geometry.grow_pv));
      } else {
        const txnid_t next_txnid = safe64_txnid_next(recent.txnid);
        if (unlikely(next_txnid > MAX_TXNID)) {
          ERROR("txnid overflow, raise %d", MDBX_TXN_FULL);
          return MDBX_TXN_FULL;
        }
        NOTICE("updating meta.geo: "
               "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN "), "
               "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN ")",
               recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper,
               pv2pages(recent.ptr_c->geometry.shrink_pv), pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid,
               header.geometry.lower, header.geometry.now, header.geometry.upper, pv2pages(header.geometry.shrink_pv),
               pv2pages(header.geometry.grow_pv), next_txnid);

        if (unlikely(header.unsafe_txnid != recent.txnid)) {
          const pgno_t recent_pgno = bytes2pgno(env, ptr_dist(recent.ptr_c, env->dxb_mmap.base));
          ERROR("meta[%u] recent steady txnid %" PRIaTXN " != header txnid %" PRIaTXN
                ", this is too unexpected and requires manual analysis.",
                recent_pgno, recent.txnid, header.unsafe_txnid);
          return MDBX_PROBLEM;
        }
        meta_set_txnid(env, &header, next_txnid);
        err = dxb_sync_locked(env, env->flags | txn_shrink_allowed, &header, &troika);
        if (err) {
          ERROR("error %d, while updating meta.geo: "
                "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN "), "
                "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN ")",
                err, recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper,
                pv2pages(recent.ptr_c->geometry.shrink_pv), pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid,
                header.geometry.lower, header.geometry.now, header.geometry.upper, pv2pages(header.geometry.shrink_pv),
                pv2pages(header.geometry.grow_pv), header.unsafe_txnid);
          return err;
        }
      }
    }

    atomic_store32(&env->lck->discarded_tail, bytes2pgno(env, allocated_aligned2os_bytes), mo_Relaxed);

    if ((env->flags & MDBX_RDONLY) == 0 && env->stuck_meta < 0 &&
        (globals.runtime_flags & MDBX_DBG_DONT_UPGRADE) == 0) {
      for (unsigned n = 0; n < NUM_METAS; ++n) {
        meta_t *const meta = METAPAGE(env, n);
        if (unlikely(unaligned_peek_u64(4, &meta->magic_and_version) != MDBX_DATA_MAGIC) ||
            (meta->dxbid.x | meta->dxbid.y) == 0 || (meta->gc_flags & ~DB_PERSISTENT_FLAGS)) {
          const txnid_t txnid = meta_is_used(&troika, n) ? constmeta_txnid(meta) : 0;
          NOTICE("%s %s"
                 "meta[%u], txnid %" PRIaTXN,
                 "updating db-format/guid signature for", meta_is_steady(meta) ? "stead-" : "weak-", n, txnid);
          err = meta_override(env, n, txnid, meta);
          if (unlikely(err != MDBX_SUCCESS) &&
              /* Just ignore the MDBX_PROBLEM error, since here it is
               * returned only in case of the attempt to upgrade an obsolete
               * meta-page that is invalid for current state of a DB,
               * e.g. after shrinking DB file */
              err != MDBX_PROBLEM) {
            ERROR("%s meta[%u], txnid %" PRIaTXN ", error %d", "updating db-format signature for", n, txnid, err);
            return err;
          }
          troika = meta_tap(env);
        }
      }
    }
  } /* lck exclusive, lck_rc == MDBX_RESULT_TRUE */

  //---------------------------------------------------- setup madvise/readahead
  if (allocated_aligned2os_bytes < env->dxb_mmap.current) {
#if defined(MADV_REMOVE)
    if (lck_rc && (env->flags & MDBX_WRITEMAP) != 0 &&
        /* not recovery mode */ env->stuck_meta < 0) {
      NOTICE("open-MADV_%s %u..%u", "REMOVE (deallocate file space)", env->lck->discarded_tail.weak,
             bytes2pgno(env, env->dxb_mmap.current));
      err = madvise(ptr_disp(env->dxb_mmap.base, allocated_aligned2os_bytes),
                    env->dxb_mmap.current - allocated_aligned2os_bytes, MADV_REMOVE)
                ? ignore_enosys_and_eagain(errno)
                : MDBX_SUCCESS;
      if (unlikely(MDBX_IS_ERROR(err)))
        return err;
    }
#endif /* MADV_REMOVE */
#if defined(MADV_DONTNEED)
    NOTICE("open-MADV_%s %u..%u", "DONTNEED", env->lck->discarded_tail.weak, bytes2pgno(env, env->dxb_mmap.current));
    err = madvise(ptr_disp(env->dxb_mmap.base, allocated_aligned2os_bytes),
                  env->dxb_mmap.current - allocated_aligned2os_bytes, MADV_DONTNEED)
              ? ignore_enosys_and_eagain(errno)
              : MDBX_SUCCESS;
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(POSIX_MADV_DONTNEED)
    err = ignore_enosys(posix_madvise(ptr_disp(env->dxb_mmap.base, allocated_aligned2os_bytes),
                                      env->dxb_mmap.current - allocated_aligned2os_bytes, POSIX_MADV_DONTNEED));
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#elif defined(POSIX_FADV_DONTNEED)
    err = ignore_enosys(posix_fadvise(env->lazy_fd, allocated_aligned2os_bytes,
                                      env->dxb_mmap.current - allocated_aligned2os_bytes, POSIX_FADV_DONTNEED));
    if (unlikely(MDBX_IS_ERROR(err)))
      return err;
#endif /* MADV_DONTNEED */
  }

  err = dxb_set_readahead(env, bytes2pgno(env, allocated_bytes), readahead, true);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  return rc;
}

int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, troika_t *const troika) {
  eASSERT0(env, ((env->flags ^ flags) & MDBX_WRITEMAP) == 0);
  eASSERT0(env, pending->trees.gc.flags == MDBX_INTEGERKEY);
  eASSERT0(env, check_table_flags(pending->trees.main.flags));
  const meta_t *const meta0 = METAPAGE(env, 0);
  const meta_t *const meta1 = METAPAGE(env, 1);
  const meta_t *const meta2 = METAPAGE(env, 2);
  const meta_ptr_t head = meta_recent(env, troika);
  int rc;

  eASSERT0(env, pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS));
  eASSERT0(env, (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0);
  eASSERT0(env, pending->geometry.first_unallocated <= pending->geometry.now);

  if (flags & MDBX_SAFE_NOSYNC) {
    /* Check auto-sync conditions */
    const pgno_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed);
    const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed);
    uint64_t eoos_timestamp;
    if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) ||
        (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) &&
         osal_monotime() - eoos_timestamp >= autosync_period))
      flags &= MDBX_WRITEMAP | txn_shrink_allowed; /* force steady */
  }

  pgno_t shrink = 0;
  if (flags & txn_shrink_allowed) {
    const size_t prev_discarded_pgno = atomic_load32(&env->lck->discarded_tail, mo_Relaxed);
    if (prev_discarded_pgno < pending->geometry.first_unallocated)
      env->lck->discarded_tail.weak = pending->geometry.first_unallocated;
    else if (prev_discarded_pgno >= pending->geometry.first_unallocated + env->madv_threshold) {
      /* LY: check conditions to discard unused pages */
      const pgno_t largest_pgno =
          mvcc_snapshot_largest(env, (head.ptr_c->geometry.first_unallocated > pending->geometry.first_unallocated)
                                         ? head.ptr_c->geometry.first_unallocated
                                         : pending->geometry.first_unallocated);
      eASSERT0(env, largest_pgno >= NUM_METAS);

#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
      const pgno_t edge = env->poison_edge;
      if (edge > largest_pgno) {
        env->poison_edge = largest_pgno;
        VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)),
                                   pgno2bytes(env, edge - largest_pgno));
        MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)),
                                       pgno2bytes(env, edge - largest_pgno));
      }
#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */

#if defined(MADV_DONTNEED) || defined(POSIX_MADV_DONTNEED)
      const size_t discard_edge_pgno = pgno_ceil2os_pgno(env, largest_pgno);
      if (prev_discarded_pgno >= discard_edge_pgno + env->madv_threshold) {
        const size_t prev_discarded_bytes = pgno_ceil2os_bytes(env, prev_discarded_pgno);
        const size_t discard_edge_bytes = pgno2bytes(env, discard_edge_pgno);
        /* из-за выравнивания prev_discarded_bytes и discard_edge_bytes
         * могут быть равны */
        if (prev_discarded_bytes > discard_edge_bytes) {
          NOTICE("shrink-MADV_%s %zu..%zu", "DONTNEED", discard_edge_pgno, prev_discarded_pgno);
          munlock_after(env, discard_edge_pgno, bytes_ceil2os_bytes(env, env->dxb_mmap.current));
          const uint32_t munlocks_before = atomic_load32(&env->lck->mlcnt[1], mo_Relaxed);
#if defined(MADV_DONTNEED)
          int advise = MADV_DONTNEED;
#if defined(MADV_FREE) && 0 /* MADV_FREE works for only anonymous vma at the moment */
          if ((env->flags & MDBX_WRITEMAP) && global.linux_kernel_version > 0x04050000)
            advise = MADV_FREE;
#endif /* MADV_FREE */
          int err = madvise(ptr_disp(env->dxb_mmap.base, discard_edge_bytes), prev_discarded_bytes - discard_edge_bytes,
                            advise)
                        ? ignore_enosys_and_eagain(errno)
                        : MDBX_SUCCESS;
#else
          int err = ignore_enosys(posix_madvise(ptr_disp(env->dxb_mmap.base, discard_edge_bytes),
                                                prev_discarded_bytes - discard_edge_bytes, POSIX_MADV_DONTNEED));
#endif
          if (unlikely(MDBX_IS_ERROR(err))) {
            const uint32_t mlocks_after = atomic_load32(&env->lck->mlcnt[0], mo_Relaxed);
            if (err == MDBX_EINVAL) {
              const int severity = (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN;
              if (LOG_ENABLED(severity))
                debug_log(severity, __func__, __LINE__,
                          "%s-madvise: ignore EINVAL (%d) since some pages maybe "
                          "locked (%u/%u mlcnt-processes)",
                          "shrink", err, mlocks_after, munlocks_before);
            } else {
              ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", "shrink", "DONTNEED",
                    discard_edge_bytes, prev_discarded_bytes - discard_edge_bytes, mlocks_after, munlocks_before, err);
              return err;
            }
          } else
            env->lck->discarded_tail.weak = discard_edge_pgno;
        }
      }
#endif /* MADV_DONTNEED || POSIX_MADV_DONTNEED */

      /* LY: check conditions to shrink datafile */
      const pgno_t stockpile_gap = 3 + pending->trees.gc.height * 3;
      pgno_t shrink_step = 0;
      if (pending->geometry.shrink_pv && pending->geometry.now - pending->geometry.first_unallocated >
                                             (shrink_step = pv2pages(pending->geometry.shrink_pv)) + stockpile_gap) {
        if (pending->geometry.now > largest_pgno &&
            pending->geometry.now - largest_pgno > shrink_step + stockpile_gap) {
          const pgno_t aligner =
              pending->geometry.grow_pv ? /* grow_step */ pv2pages(pending->geometry.grow_pv) : shrink_step;
          const pgno_t with_stockpile_gap = largest_pgno + stockpile_gap;
          const pgno_t aligned =
              pgno_ceil2os_pgno(env, (size_t)with_stockpile_gap + aligner - with_stockpile_gap % aligner);
          const pgno_t bottom = (aligned > pending->geometry.lower) ? aligned : pending->geometry.lower;
          if (pending->geometry.now > bottom) {
            if (TROIKA_HAVE_STEADY(troika))
              /* force steady, but only if steady-checkpoint is present */
              flags &= MDBX_WRITEMAP | txn_shrink_allowed;
            shrink = pending->geometry.now - bottom;
            pending->geometry.now = bottom;
            if (unlikely(head.txnid == pending->unsafe_txnid)) {
              const txnid_t txnid = safe64_txnid_next(pending->unsafe_txnid);
              NOTICE("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN, pending->unsafe_txnid, txnid);
              ENSURE_OBJ(env, !env->basal_txn || !env->txn);
              if (unlikely(txnid > MAX_TXNID)) {
                rc = MDBX_TXN_FULL;
                ERROR("txnid overflow, raise %d", rc);
                goto fail;
              }
              meta_set_txnid(env, pending, txnid);
              eASSERT1(env, coherency_check_meta(env, pending, true));
            }
          }
        }
      }
    }
  }

  /* LY: step#1 - sync previously written/updated data-pages */
  rc = MDBX_RESULT_FALSE /* carry steady */;
  const uint64_t snap_unsynced_pages = atomic_load64(&env->lck->unsynced_pages, mo_Relaxed);
  if (snap_unsynced_pages) {
    if (snap_unsynced_pages == 1 && (flags & MDBX_NOMETASYNC))
      goto skip_lastmeta_sync;

    eASSERT0(env, ((flags ^ env->flags) & MDBX_WRITEMAP) == 0);
    enum osal_syncmode_bits mode_bits = MDBX_SYNC_NONE;
    if ((flags & MDBX_SAFE_NOSYNC) == 0) {
      mode_bits = MDBX_SYNC_DATA;
      if (pending->geometry.first_unallocated > meta_prefer_steady(env, troika).ptr_c->geometry.now)
        mode_bits |= MDBX_SYNC_SIZE;
      if (flags & MDBX_NOMETASYNC)
        mode_bits |= MDBX_SYNC_IODQ;
    } else if (unlikely(env->incore))
      goto skip_incore_sync;

    if (flags & MDBX_WRITEMAP)
      rc = dxb_msync(env, pending->geometry.first_unallocated, mode_bits);
    else
      rc = dxb_fsync(env, mode_bits);
    if (unlikely(rc != MDBX_SUCCESS))
      goto fail;
    rc = (flags & MDBX_SAFE_NOSYNC) ? MDBX_RESULT_TRUE /* carry non-steady */
                                    : MDBX_RESULT_FALSE /* carry steady */;
  }
  eASSERT1(env, coherency_check_meta(env, pending, true));

  /* Steady or Weak */
  if (rc == MDBX_RESULT_FALSE /* carry steady */) {
    meta_sign_as_steady(pending);
    atomic_store64(&env->lck->eoos_timestamp, 0, mo_Relaxed);
    atomic_store64(&env->lck->unsynced_pages, 0, mo_Relaxed);
  } else {
    ASSERT(rc == MDBX_RESULT_TRUE /* carry non-steady */);
  skip_incore_sync:
    eASSERT0(env, env->lck->unsynced_pages.weak > 0);
    /* Может быть нулевым если unsynced_pages > 0 в результате спиллинга.
     * eASSERT0(env, env->lck->eoos_timestamp.weak != 0); */
    unaligned_poke_u64(4, pending->sign, DATASIGN_WEAK);
  skip_lastmeta_sync:;
  }

  const bool legal4overwrite = head.txnid == pending->unsafe_txnid &&
                               !memcmp(&head.ptr_c->trees, &pending->trees, sizeof(pending->trees)) &&
                               !memcmp(&head.ptr_c->canary, &pending->canary, sizeof(pending->canary)) &&
                               !memcmp(&head.ptr_c->geometry, &pending->geometry, sizeof(pending->geometry));
  meta_t *target = nullptr;
  if (head.txnid == pending->unsafe_txnid) {
    ENSURE_OBJ(env, legal4overwrite);
    if (!head.is_steady && meta_is_steady(pending))
      target = (meta_t *)head.ptr_c;
    else {
      NOTICE("skip update meta%" PRIaPGNO " for txn#%" PRIaTXN ", since it is already steady",
             payload2page(head.ptr_c)->pgno, head.txnid);
      return MDBX_SUCCESS;
    }
  } else {
    const unsigned troika_tail = troika->tail_and_flags & 3;
    ENSURE_OBJ(env, troika_tail < NUM_METAS && troika_tail != troika->recent && troika_tail != troika->prefer_steady);
    target = (meta_t *)meta_tail(env, troika).ptr_c;
  }

  /* LY: step#2 - update meta-page. */
  DEBUG("writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO
        "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s",
        payload2page(target)->pgno, pending->trees.main.root, pending->trees.gc.root, pending->geometry.lower,
        pending->geometry.first_unallocated, pending->geometry.now, pending->geometry.upper,
        pv2pages(pending->geometry.grow_pv), pv2pages(pending->geometry.shrink_pv), pending->unsafe_txnid,
        durable_caption(pending));

  DEBUG("meta0: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO,
        (meta0 == head.ptr_c) ? "head"
        : (meta0 == target)   ? "tail"
                              : "stay",
        durable_caption(meta0), constmeta_txnid(meta0), meta0->trees.main.root, meta0->trees.gc.root);
  DEBUG("meta1: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO,
        (meta1 == head.ptr_c) ? "head"
        : (meta1 == target)   ? "tail"
                              : "stay",
        durable_caption(meta1), constmeta_txnid(meta1), meta1->trees.main.root, meta1->trees.gc.root);
  DEBUG("meta2: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO,
        (meta2 == head.ptr_c) ? "head"
        : (meta2 == target)   ? "tail"
                              : "stay",
        durable_caption(meta2), constmeta_txnid(meta2), meta2->trees.main.root, meta2->trees.gc.root);

  eASSERT1(env, pending->unsafe_txnid != constmeta_txnid(meta0) || (meta_is_steady(pending) && !meta_is_steady(meta0)));
  eASSERT1(env, pending->unsafe_txnid != constmeta_txnid(meta1) || (meta_is_steady(pending) && !meta_is_steady(meta1)));
  eASSERT1(env, pending->unsafe_txnid != constmeta_txnid(meta2) || (meta_is_steady(pending) && !meta_is_steady(meta2)));

  eASSERT0(env, ((env->flags ^ flags) & MDBX_WRITEMAP) == 0);
  ENSURE_OBJ(env, target == head.ptr_c || constmeta_txnid(target) < pending->unsafe_txnid);
  if (flags & MDBX_WRITEMAP) {
    jitter4testing(true);
    if (likely(target != head.ptr_c)) {
      /* LY: 'invalidate' the meta. */
      meta_update_begin(env, target, pending->unsafe_txnid);
      unaligned_poke_u64(4, target->sign, DATASIGN_WEAK);
#if MDBX_CHECKING > 0 || MDBX_DEBUG > 0
      /* debug: provoke failure to catch a violators, but don't touch pagesize
       * to allow readers catch actual pagesize. */
      void *provoke_begin = &target->trees.gc.root;
      void *provoke_end = &target->sign;
      memset(provoke_begin, 0xCC, ptr_dist(provoke_end, provoke_begin));
      jitter4testing(false);
#endif /* MDBX_CHECKING > 0 || MDBX_DEBUG > 0 */

      /* LY: update info */
      target->geometry = pending->geometry;
      target->trees.gc = pending->trees.gc;
      target->trees.main = pending->trees.main;
      eASSERT0(env, target->trees.gc.flags == MDBX_INTEGERKEY);
      eASSERT0(env, check_table_flags(target->trees.main.flags));
      target->canary = pending->canary;
      memcpy(target->pages_retired, pending->pages_retired, 8);
      jitter4testing(true);

      /* LY: 'commit' the meta */
      meta_update_end(env, target, unaligned_peek_u64(4, pending->txnid_b));
      jitter4testing(true);
      eASSERT1(env, coherency_check_meta(env, target, true));
    } else {
      /* dangerous case (target == head), only sign could
       * me updated, check assertions once again */
      eASSERT0(env, legal4overwrite && !head.is_steady && meta_is_steady(pending));
    }
    memcpy(target->sign, pending->sign, 8);
    osal_flush_incoherent_cpu_writeback();
    jitter4testing(true);
    if (!env->incore) {
      if (!MDBX_AVOID_MSYNC)
        /* sync meta-pages */
        rc = dxb_msync(env, NUM_METAS, (flags & MDBX_NOMETASYNC) ? MDBX_SYNC_NONE : MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
      else {
        if (MDBX_ENABLE_PGOP_STAT)
          env->lck->pgops.wops.weak += 1;
        const page_t *page = payload2page(target);
        rc = osal_pwrite(env->fd4meta, page, env->ps, ptr_dist(page, env->dxb_mmap.base));
        if (likely(rc == MDBX_SUCCESS)) {
          osal_flush_incoherent_mmap(target, sizeof(meta_t), globals.sys_pagesize);
          if ((flags & MDBX_NOMETASYNC) == 0 && env->fd4meta == env->lazy_fd)
            rc = dxb_fsync(env, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
        }
      }
      if (unlikely(rc != MDBX_SUCCESS))
        goto fail;
    }
  } else {
    if (MDBX_ENABLE_PGOP_STAT)
      env->lck->pgops.wops.weak += 1;
    const meta_t undo_meta = *target;
    eASSERT0(env, pending->trees.gc.flags == MDBX_INTEGERKEY);
    eASSERT0(env, check_table_flags(pending->trees.main.flags));
    rc = osal_pwrite(env->fd4meta, pending, sizeof(meta_t), ptr_dist(target, env->dxb_mmap.base));
    if (unlikely(rc != MDBX_SUCCESS)) {
    undo:
      DEBUG("%s", "write failed, disk error?");
      /* On a failure, the pagecache still contains the new data.
       * Try write some old data back, to prevent it from being used. */
      osal_pwrite(env->fd4meta, &undo_meta, sizeof(meta_t), ptr_dist(target, env->dxb_mmap.base));
      goto fail;
    }
    osal_flush_incoherent_mmap(target, sizeof(meta_t), globals.sys_pagesize);
    /* sync meta-pages */
    if (env->fd4meta == env->lazy_fd && !env->incore) {
      if (flags & MDBX_NOMETASYNC)
        env->lck->unsynced_pages.weak += 1;
      else {
        rc = dxb_fsync(env, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
        if (rc != MDBX_SUCCESS)
          goto undo;
      }
    }
  }

  uint64_t timestamp = 0;
  /* coverity[array_null] */
  while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") {
    rc = coherency_check_written(env, pending->unsafe_txnid, target,
                                 bytes2pgno(env, ptr_dist(target, env->dxb_mmap.base)), &timestamp);
    if (likely(rc == MDBX_SUCCESS))
      break;
    if (unlikely(rc != MDBX_RESULT_TRUE))
      goto fail;
  }

  const uint32_t sync_txnid_dist = ((flags & MDBX_NOMETASYNC) == 0)                     ? 0
                                   : ((flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) ? MDBX_NOMETASYNC_LAZY_FD
                                                                                        : MDBX_NOMETASYNC_LAZY_WRITEMAP;
  env->lck->meta_sync_txnid.weak = pending->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__].weak - sync_txnid_dist;

  *troika = meta_tap(env);
  for (MDBX_txn *txn = env->basal_txn; txn; txn = txn->nested)
    if (troika != &txn->wr.troika)
      txn->wr.troika = *troika;

  /* LY: shrink datafile if needed */
  if (unlikely(shrink)) {
    VERBOSE("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")", pending->geometry.now, shrink);
    rc = dxb_resize(env, pending->geometry.first_unallocated, pending->geometry.now, pending->geometry.upper,
                    impilict_shrink);
    if (rc != MDBX_SUCCESS && rc != MDBX_EPERM)
      goto fail;
    eASSERT1(env, coherency_check_meta(env, target, true));
  }

  lck_t *const lck = env->lck_mmap.lck;
  if (likely(lck))
    /* toggle oldest refresh */
    atomic_store32(&lck->rdt_refresh_flag, false, mo_Relaxed);

  return MDBX_SUCCESS;

fail:
  env->flags |= ENV_FATAL_ERROR;
  return rc;
}

MDBX_txn *env_owned_wrtxn(const MDBX_env *env) {
  if (likely(env->basal_txn)) {
    const bool is_owned = (env->flags & MDBX_NOSTICKYTHREADS) ? (env->basal_txn->owner != 0)
                                                              : (env->basal_txn->owner == osal_thread_self());
    if (is_owned)
      return env->txn ? env->txn : env->basal_txn;
  }
  return nullptr;
}

int env_page_auxbuffer(MDBX_env *env) {
  const int err = env->page_auxbuf
                      ? MDBX_SUCCESS
                      : osal_memalign_alloc(globals.sys_pagesize, env->ps * (size_t)NUM_METAS, &env->page_auxbuf);
  if (likely(err == MDBX_SUCCESS)) {
    memset(env->page_auxbuf, -1, env->ps * (size_t)2);
    memset(ptr_disp(env->page_auxbuf, env->ps * (size_t)2), 0, env->ps);
  }
  return err;
}

__cold unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize) {
  STATIC_ASSERT(PTRDIFF_MAX > MAX_MAPSIZE);
  STATIC_ASSERT(MDBX_MIN_PAGESIZE > sizeof(page_t) + sizeof(meta_t));
  ENSURE_OBJ(env, is_powerof2(pagesize));
  ENSURE_OBJ(env, pagesize >= MDBX_MIN_PAGESIZE);
  ENSURE_OBJ(env, pagesize <= MDBX_MAX_PAGESIZE);
  ENSURE_OBJ(env, !env->page_auxbuf && env->ps != pagesize);
  env->ps = (unsigned)pagesize;

  STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MIN_PAGESIZE) > 4);
  STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MAX_PAGESIZE) < PAGELIST_LIMIT);
  const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
  ENSURE_OBJ(env, maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)PAGELIST_LIMIT / 4);
  env->maxgc_large1page = (unsigned)maxgc_ov1page;
  env->maxgc_per_branch = (unsigned)((pagesize - PAGEHDRSZ) / (sizeof(indx_t) + sizeof(node_t) + sizeof(txnid_t)));

  STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) > sizeof(tree_t) + NODESIZE + 42);
  STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX);
  STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) >= BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE));
  STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) > NODESIZE + 42);
  STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX);
  const intptr_t branch_nodemax = BRANCH_NODE_MAX(pagesize);
  const intptr_t leaf_nodemax = LEAF_NODE_MAX(pagesize);
  ENSURE_OBJ(env, branch_nodemax > (intptr_t)(NODESIZE + 42) && branch_nodemax % 2 == 0 &&
                      leaf_nodemax > (intptr_t)(sizeof(tree_t) + NODESIZE + 42) && leaf_nodemax >= branch_nodemax &&
                      leaf_nodemax < (int)UINT16_MAX && leaf_nodemax % 2 == 0);
  env->leaf_nodemax = (uint16_t)leaf_nodemax;
  env->branch_nodemax = (uint16_t)branch_nodemax;
  env->ps2ln = (uint8_t)log2n_powerof2(pagesize);
  eASSERT0(env, pgno2bytes(env, 1) == pagesize);
  eASSERT0(env, bytes2pgno(env, pagesize + pagesize) == 2);
  recalculate_merge_thresholds(env);
  recalculate_subpage_thresholds(env);
  env_options_adjust_dp_limit(env);
  return env->ps;
}

__cold int env_sync(MDBX_env *env, bool force, bool nonblock) {
  if (unlikely(env->flags & MDBX_RDONLY))
    return MDBX_EACCESS;

  MDBX_txn *const txn_owned = env_owned_wrtxn(env);
  bool should_unlock = false;
  int rc = MDBX_RESULT_TRUE /* means "nothing to sync" */;

retry:;
  unsigned flags = env->flags & ~(MDBX_NOMETASYNC | txn_shrink_allowed);
  if (unlikely((flags & (ENV_FATAL_ERROR | ENV_ACTIVE)) != ENV_ACTIVE)) {
    rc = (flags & ENV_FATAL_ERROR) ? MDBX_PANIC : MDBX_EPERM;
    goto bailout;
  }

  const troika_t troika = (txn_owned || should_unlock) ? env->basal_txn->wr.troika : meta_tap(env);
  const meta_ptr_t head = meta_recent(env, &troika);
  const uint64_t unsynced_pages = atomic_load64(&env->lck->unsynced_pages, mo_Relaxed);
  if (unsynced_pages == 0) {
    const uint32_t synched_meta_txnid_u32 = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed);
    if (synched_meta_txnid_u32 == (uint32_t)head.txnid && head.is_steady)
      goto bailout;
  }

  if (should_unlock && (env->flags & MDBX_WRITEMAP) &&
      unlikely(head.ptr_c->geometry.first_unallocated > bytes2pgno(env, env->dxb_mmap.current))) {

    if (unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) {
      NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent "
             "meta-page (%u)",
             "sync datafile", env->stuck_meta, troika.recent);
      rc = MDBX_RESULT_TRUE;
    } else {
      rc = dxb_resize(env, head.ptr_c->geometry.first_unallocated, head.ptr_c->geometry.now, head.ptr_c->geometry.upper,
                      implicit_grow);
      if (unlikely(rc != MDBX_SUCCESS))
        goto bailout;
    }
  }

  const size_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed);
  const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed);
  uint64_t eoos_timestamp;
  if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) ||
      (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) &&
       osal_monotime() - eoos_timestamp >= autosync_period))
    flags &= MDBX_WRITEMAP /* clear flags for full steady sync */;

  if (!txn_owned) {
    if (!should_unlock) {
      unsigned wops = 0;

      int err;
      /* pre-sync to avoid latency for writer */
      if (unsynced_pages > /* FIXME: define threshold */ 42 && (flags & MDBX_SAFE_NOSYNC) == 0) {
        eASSERT0(env, ((flags ^ env->flags) & MDBX_WRITEMAP) == 0);
        if (flags & MDBX_WRITEMAP) {
          /* Acquire guard to avoid collision with remap */
#if defined(_WIN32) || defined(_WIN64)
          imports.srwl_AcquireShared(&env->remap_lock);
#else
          err = osal_fastmutex_acquire(&env->remap_lock);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
#endif
          err = dxb_msync(env, head.ptr_c->geometry.first_unallocated, MDBX_SYNC_DATA);
#if defined(_WIN32) || defined(_WIN64)
          imports.srwl_ReleaseShared(&env->remap_lock);
#else
          int unlock_err = osal_fastmutex_release(&env->remap_lock);
          if (unlikely(unlock_err != MDBX_SUCCESS) && err == MDBX_SUCCESS)
            err = unlock_err;
#endif
        } else
          err = dxb_fsync(env, MDBX_SYNC_DATA);

        if (unlikely(err != MDBX_SUCCESS))
          return err;

        wops = 1;
        /* pre-sync done */
        rc = MDBX_SUCCESS /* means "some data was synced" */;
      }

      err = lck_txn_lock(env, nonblock);
      if (unlikely(err != MDBX_SUCCESS))
        return err;

      should_unlock = true;
      if (MDBX_ENABLE_PGOP_STAT)
        env->lck->pgops.wops.weak += wops;
      env->basal_txn->wr.troika = meta_tap(env);
      eASSERT0(env, !env->txn && !env->basal_txn->nested);
      goto retry;
    }
    eASSERT1(env, head.txnid == recent_committed_txnid(env));
    env->basal_txn->txnid = head.txnid;
    txn_gc_detent(env->basal_txn);
    flags |= txn_shrink_allowed;
  }

  eASSERT0(env, txn_owned || should_unlock);
  eASSERT0(env, !txn_owned || (flags & txn_shrink_allowed) == 0);

  if (!head.is_steady && unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) {
    NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent "
           "meta-page (%u)",
           "sync datafile", env->stuck_meta, troika.recent);
    rc = MDBX_RESULT_TRUE;
    goto bailout;
  }
  if (!head.is_steady || ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) {
    DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, payload2page(head.ptr_c)->pgno,
          durable_caption(head.ptr_c), unsynced_pages);
    meta_t meta = *head.ptr_c;
    rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->wr.troika);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;
  }

  /* LY: sync meta-pages if MDBX_NOMETASYNC enabled
   *     and someone was not synced above. */
  if (atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid)
    rc = meta_sync(env, head);

bailout:
  if (should_unlock)
    lck_txn_unlock(env);
  return rc;
}

__cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
  /* Использование O_DSYNC или FILE_FLAG_WRITE_THROUGH:
   *
   *   0) Если размер страниц БД меньше системной страницы ОЗУ, то ядру ОС
   *      придется чаще обновлять страницы в unified page cache.
   *
   *      Однако, O_DSYNC не предполагает отключение unified page cache,
   *      поэтому подобные затруднения будем считать проблемой ОС и/или
   *      ожидаемым пенальти из-за использования мелких страниц БД.
   *
   *   1) В режиме MDBX_SYNC_DURABLE - O_DSYNC для записи как данных,
   *      так и мета-страниц. Однако, на Linux отказ от O_DSYNC с последующим
   *      fdatasync() может быть выгоднее при использовании HDD, так как
   *      позволяет io-scheduler переупорядочить запись с учетом актуального
   *      расположения файла БД на носителе.
   *
   *   2) В режиме MDBX_NOMETASYNC - O_DSYNC можно использовать для данных,
   *      но в этом может не быть смысла, так как fdatasync() всё равно
   *      требуется для гарантии фиксации мета после предыдущей транзакции.
   *
   *      В итоге на нормальных системах (не Windows) есть два варианта:
   *       - при возможности O_DIRECT и/или io_ring для данных, скорее всего,
   *         есть смысл вызвать fdatasync() перед записью данных, а затем
   *         использовать O_DSYNC;
   *       - не использовать O_DSYNC и вызывать fdatasync() после записи данных.
   *
   *      На Windows же следует минимизировать использование FlushFileBuffers()
   *      из-за проблем с производительностью. Поэтому на Windows в режиме
   *      MDBX_NOMETASYNC:
   *       - мета обновляется через дескриптор без FILE_FLAG_WRITE_THROUGH;
   *       - перед началом записи данных вызывается FlushFileBuffers(), если
   *         meta_sync_txnid не совпадает с последней записанной мета;
   *       - данные записываются через дескриптор с FILE_FLAG_WRITE_THROUGH.
   *
   *   3) В режиме MDBX_SAFE_NOSYNC - O_DSYNC нет смысла использовать, пока не
   *      будет реализована возможность полностью асинхронной "догоняющей"
   *      записи в выделенном процессе-сервере с io-ring очередями внутри.
   *
   * -----
   *
   * Использование O_DIRECT или FILE_FLAG_NO_BUFFERING:
   *
   *   Назначение этих флагов в отключении файлового дескриптора от
   *   unified page cache, т.е. от отображенных в память данных в случае
   *   libmdbx.
   *
   *   Поэтому, использование direct i/o в libmdbx без MDBX_WRITEMAP лишено
   *   смысла и контр-продуктивно, ибо так мы провоцируем ядро ОС на
   *   не-когерентность отображения в память с содержимым файла на носителе,
   *   либо требуем дополнительных проверок и действий направленных на
   *   фактическое отключение O_DIRECT для отображенных в память данных.
   *
   *   В режиме MDBX_WRITEMAP когерентность отображенных данных обеспечивается
   *   физически. Поэтому использование direct i/o может иметь смысл, если у
   *   ядра ОС есть какие-то проблемы с msync(), в том числе с
   *   производительностью:
   *    - использование io_ring или gather-write может быть дешевле, чем
   *      просмотр PTE ядром и запись измененных/грязных;
   *    - но проблема в том, что записываемые из user mode страницы либо не
   *      будут помечены чистыми (и соответственно будут записаны ядром
   *      еще раз), либо ядру необходимо искать и чистить PTE при получении
   *      запроса на запись.
   *
   *   Поэтому O_DIRECT или FILE_FLAG_NO_BUFFERING используется:
   *    - только в режиме MDBX_SYNC_DURABLE с MDBX_WRITEMAP;
   *    - когда ps >= me_os_psize;
   *    - опция сборки MDBX_AVOID_MSYNC != 0, которая по-умолчанию включена
   *      только на Windows (см ниже).
   *
   * -----
   *
   * Использование FILE_FLAG_OVERLAPPED на Windows:
   *
   * У Windows очень плохо с I/O (за исключением прямых постраничных
   * scatter/gather, которые работают в обход проблемного unified page
   * cache и поэтому почти бесполезны в libmdbx).
   *
   * При этом всё еще хуже при использовании FlushFileBuffers(), что также
   * требуется после FlushViewOfFile() в режиме MDBX_WRITEMAP. Поэтому
   * на Windows вместо FlushViewOfFile() и FlushFileBuffers() следует
   * использовать запись через дескриптор с FILE_FLAG_WRITE_THROUGH.
   *
   * В свою очередь, запись с FILE_FLAG_WRITE_THROUGH дешевле/быстрее
   * при использовании FILE_FLAG_OVERLAPPED. В результате, на Windows
   * в durable-режимах запись данных всегда в overlapped-режиме,
   * при этом для записи мета требуется отдельный не-overlapped дескриптор.
   */

  env->pid = osal_getpid();
  int rc = osal_openfile((env->flags & MDBX_RDONLY) ? MDBX_OPEN_DXB_READ : MDBX_OPEN_DXB_LAZY, env, env->pathname.dxb,
                         &env->lazy_fd, mode);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

#if MDBX_LOCKING == MDBX_LOCKING_SYSV
  env->me_sysv_ipc.key = ftok(env->pathname.dxb, 42);
  if (unlikely(env->me_sysv_ipc.key == -1))
    return errno;
#endif /* MDBX_LOCKING */

  /* Set the position in files outside of the data to avoid corruption
   * due to erroneous use of file descriptors in the application code. */
  const uint64_t safe_parking_lot_offset = UINT64_C(0x7fffFFFF80000000);
  osal_fseek(env->lazy_fd, safe_parking_lot_offset);

  env->fd4meta = env->lazy_fd;
#if defined(_WIN32) || defined(_WIN64)
  env->dxb_lock_event = CreateEventW(nullptr, true, false, nullptr);
  if (unlikely(!env->dxb_lock_event))
    return (int)GetLastError();
  env->lck_lock_event = CreateEventW(nullptr, true, false, nullptr);
  if (unlikely(!env->lck_lock_event))
    return (int)GetLastError();
  eASSERT0(env, env->ioring.overlapped_fd == 0);
  bool ior_direct = false;
  if (!(env->flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_EXCLUSIVE))) {
    if (MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP)) {
      /* Запрошен режим MDBX_SYNC_DURABLE | MDBX_WRITEMAP при активной опции
       * MDBX_AVOID_MSYNC.
       *
       * 1) В этой комбинации наиболее выгодно использовать WriteFileGather(),
       * но для этого необходимо открыть файл с флагом FILE_FLAG_NO_BUFFERING и
       * после обеспечивать выравнивание адресов и размера данных на границу
       * системной страницы, что в свою очередь возможно если размер страницы БД
       * не меньше размера системной страницы ОЗУ. Поэтому для открытия файла в
       * нужном режиме требуется знать размер страницы БД.
       *
       * 2) Кроме этого, в Windows запись в заблокированный регион файла
       * возможно только через тот-же дескриптор. Поэтому изначальный захват
       * блокировок посредством lck_seize(), захват/освобождение блокировок
       * во время пишущих транзакций и запись данных должны выполнятся через
       * один дескриптор.
       *
       * Таким образом, требуется прочитать волатильный заголовок БД, чтобы
       * узнать размер страницы, чтобы открыть дескриптор файла в режиме нужном
       * для записи данных, чтобы использовать именно этот дескриптор для
       * изначального захвата блокировок. */
      meta_t header;
      uint64_t dxb_filesize;
      int err = dxb_read_header(env, &header, MDBX_SUCCESS, true);
      if ((err == MDBX_SUCCESS && header.pagesize >= globals.sys_pagesize) ||
          (err == MDBX_ENODATA && mode && env->ps >= globals.sys_pagesize &&
           osal_filesize(env->lazy_fd, &dxb_filesize) == MDBX_SUCCESS && dxb_filesize == 0))
        /* Может быть коллизия, если два процесса пытаются одновременно создать
         * БД с разным размером страницы, который у одного меньше системной
         * страницы, а у другого НЕ меньше. Эта допустимая, но очень странная
         * ситуация. Поэтому считаем её ошибочной и не пытаемся разрешить. */
        ior_direct = true;
    }

    rc = osal_openfile(ior_direct ? MDBX_OPEN_DXB_OVERLAPPED_DIRECT : MDBX_OPEN_DXB_OVERLAPPED, env, env->pathname.dxb,
                       &env->ioring.overlapped_fd, 0);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    osal_fseek(env->ioring.overlapped_fd, safe_parking_lot_offset);
  }
#else
  if (mode == 0) {
    /* pickup mode for lck-file */
    struct stat st;
    if (unlikely(fstat(env->lazy_fd, &st)))
      return errno;
    mode = st.st_mode;
  }
  mode = (/* inherit read permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
         /* always add read/write for owner */ S_IRUSR | S_IWUSR |
         ((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) |
         ((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0);
#endif /* !Windows */
  const int lck_rc = lck_setup(env, mode);
  if (unlikely(MDBX_IS_ERROR(lck_rc)))
    return lck_rc;
  if (env->lck_mmap.fd != INVALID_HANDLE_VALUE)
    osal_fseek(env->lck_mmap.fd, safe_parking_lot_offset);

  eASSERT0(env, env->dsync_fd == INVALID_HANDLE_VALUE);
  if (!(env->flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | DEPRECATED_MAPASYNC
#if defined(_WIN32) || defined(_WIN64)
                      | MDBX_EXCLUSIVE
#endif /* !Windows */
                      ))) {
    rc = osal_openfile(MDBX_OPEN_DXB_DSYNC, env, env->pathname.dxb, &env->dsync_fd, 0);
    if (unlikely(MDBX_IS_ERROR(rc)))
      return rc;
    if (env->dsync_fd != INVALID_HANDLE_VALUE) {
      if ((env->flags & MDBX_NOMETASYNC) == 0)
        env->fd4meta = env->dsync_fd;
      osal_fseek(env->dsync_fd, safe_parking_lot_offset);
    }
  }

  const MDBX_env_flags_t lazy_flags = MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_NOMETASYNC;
  const MDBX_env_flags_t mode_flags = lazy_flags | MDBX_LIFORECLAIM | MDBX_NORDAHEAD | MDBX_RDONLY | MDBX_WRITEMAP;

  lck_t *const lck = env->lck_mmap.lck;
  if (lck && lck_rc != MDBX_RESULT_TRUE && (env->flags & MDBX_RDONLY) == 0) {
    MDBX_env_flags_t snap_flags;
    while ((snap_flags = atomic_load32(&lck->envmode, mo_AcquireRelease)) == MDBX_RDONLY) {
      if (atomic_cas32(&lck->envmode, MDBX_RDONLY, (snap_flags = (env->flags & mode_flags)))) {
        /* The case:
         *  - let's assume that for some reason the DB file is smaller
         *    than it should be according to the geometry,
         *    but not smaller than the last page used;
         *  - the first process that opens the database (lck_rc == RESULT_TRUE)
         *    does this in readonly mode and therefore cannot bring
         *    the file size back to normal;
         *  - some next process (lck_rc != RESULT_TRUE) opens the DB in
         *    read-write mode and now is here.
         *
         * FIXME: Should we re-check and set the size of DB-file right here? */
        break;
      }
      atomic_yield();
    }

    if (env->flags & MDBX_ACCEDE) {
      /* Pickup current mode-flags (MDBX_LIFORECLAIM, MDBX_NORDAHEAD, etc). */
      const MDBX_env_flags_t diff =
          (snap_flags ^ env->flags) & ((snap_flags & lazy_flags) ? mode_flags : mode_flags & ~MDBX_WRITEMAP);
      env->flags ^= diff;
      NOTICE("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->flags ^ diff, env->flags);
    }

    /* Ранее упущенный не очевидный момент: При работе БД в режимах
     * не-синхронной/отложенной фиксации на диске, все процессы-писатели должны
     * иметь одинаковый режим MDBX_WRITEMAP.
     *
     * В противном случае, сброс на диск следует выполнять дважды: сначала
     * msync(), затем fdatasync(). При этом msync() не обязан отрабатывать
     * в процессах без MDBX_WRITEMAP, так как файл в память отображен только
     * для чтения. Поэтому, в общем случае, различия по MDBX_WRITEMAP не
     * позволяют выполнить фиксацию данных на диск, после их изменения в другом
     * процессе.
     *
     * В режиме MDBX_UTTERLY_NOSYNC позволять совместную работу с MDBX_WRITEMAP
     * также не следует, поскольку никакой процесс (в том числе последний) не
     * может гарантированно сбросить данные на диск, а следовательно не должен
     * помечать какую-либо транзакцию как steady.
     *
     * В результате, требуется либо запретить совместную работу процессам с
     * разным MDBX_WRITEMAP в режиме отложенной записи, либо отслеживать такое
     * смешивание и блокировать steady-пометки - что контрпродуктивно. */
    const MDBX_env_flags_t rigorous_flags = (snap_flags & lazy_flags)
                                                ? MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_WRITEMAP
                                                : MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC;
    const MDBX_env_flags_t rigorous_diff = (snap_flags ^ env->flags) & rigorous_flags;
    if (rigorous_diff) {
      ERROR("current mode/flags 0x%X incompatible with requested 0x%X, "
            "rigorous diff 0x%X",
            env->flags, snap_flags, rigorous_diff);
      return MDBX_INCOMPATIBLE;
    }
  }

  env_clear_incore_cache(env);
  const int dxb_rc = dxb_setup(env, lck_rc, mode);
  if (MDBX_IS_ERROR(dxb_rc))
    return dxb_rc;

  rc = osal_check_fs_incore(env->lazy_fd);
  env->incore = false;
  if (rc == MDBX_RESULT_TRUE) {
    env->incore = true;
    NOTICE("%s", "in-core database");
    rc = MDBX_SUCCESS;
  } else if (unlikely(rc != MDBX_SUCCESS)) {
    ERROR("check_fs_incore(), err %d", rc);
    return rc;
  }

  if (unlikely(/* recovery mode */ env->stuck_meta >= 0) &&
      (lck_rc != /* exclusive */ MDBX_RESULT_TRUE || (env->flags & MDBX_EXCLUSIVE) == 0)) {
    ERROR("%s", "recovery requires exclusive mode");
    return MDBX_BUSY;
  }

  DEBUG("opened dbenv %p", __Wpedantic_format_voidptr(env));
  env->flags |= ENV_ACTIVE;
  if (!lck || lck_rc == MDBX_RESULT_TRUE) {
    env->lck->envmode.weak = env->flags & mode_flags;
    env->lck->meta_sync_txnid.weak = (uint32_t)recent_committed_txnid(env);
    env->lck->readers_check_timestamp.weak = osal_monotime();
  }
  if (lck) {
    if (lck_rc == MDBX_RESULT_TRUE) {
      rc = lck_downgrade(env);
      DEBUG("lck-downgrade-%s: rc %i", (env->flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc);
      if (rc != MDBX_SUCCESS)
        return rc;
    } else {
      rc = mvcc_cleanup_dead(env, false, nullptr);
      if (MDBX_IS_ERROR(rc))
        return rc;
    }
  }

  rc = (env->flags & MDBX_RDONLY) ? MDBX_SUCCESS
                                  : osal_ioring_create(&env->ioring
#if defined(_WIN32) || defined(_WIN64)
                                                       ,
                                                       ior_direct, env->ioring.overlapped_fd
#endif /* Windows */
                                    );
  return rc;
}

__cold int env_close(MDBX_env *env, bool resurrect_after_fork) {
  const unsigned flags = env->flags;
  env->flags &= ~ENV_INTERNAL_FLAGS;
  if (flags & ENV_TXKEY) {
    thread_key_delete(env->me_txkey);
    env->me_txkey = 0;
  }

  if (env->lck)
    munlock_all(env);

  rthc_lock();
  int rc = rthc_remove(env);
  rthc_unlock();

#if MDBX_ENABLE_DBI_LOCKFREE
  for (defer_free_item_t *next, *ptr = env->defer_free; ptr; ptr = next) {
    next = ptr->next;
    osal_free(ptr);
  }
  env->defer_free = nullptr;
#endif /* MDBX_ENABLE_DBI_LOCKFREE */

  if ((env->flags & MDBX_RDONLY) == 0)
    osal_ioring_destroy(&env->ioring);

  env->lck = nullptr;
  if (env->lck_mmap.lck)
    osal_munmap(&env->lck_mmap);

  if (env->dxb_mmap.base) {
    osal_munmap(&env->dxb_mmap);
#ifdef ENABLE_MEMCHECK
    VALGRIND_DISCARD(env->valgrind_handle);
    env->valgrind_handle = -1;
#endif /* ENABLE_MEMCHECK */
  }

#if defined(_WIN32) || defined(_WIN64)
  eASSERT0(env, !env->ioring.overlapped_fd || env->ioring.overlapped_fd == INVALID_HANDLE_VALUE);
  if (env->dxb_lock_event != INVALID_HANDLE_VALUE) {
    CloseHandle(env->dxb_lock_event);
    env->dxb_lock_event = INVALID_HANDLE_VALUE;
  }
  if (env->lck_lock_event != INVALID_HANDLE_VALUE) {
    CloseHandle(env->lck_lock_event);
    env->lck_lock_event = INVALID_HANDLE_VALUE;
  }
  eASSERT0(env, !resurrect_after_fork);
  if (env->pathname_char) {
    osal_free(env->pathname_char);
    env->pathname_char = nullptr;
  }
#endif /* Windows */

  if (env->dsync_fd != INVALID_HANDLE_VALUE) {
    (void)osal_closefile(env->dsync_fd);
    env->dsync_fd = INVALID_HANDLE_VALUE;
  }

  if (env->lazy_fd != INVALID_HANDLE_VALUE) {
    (void)osal_closefile(env->lazy_fd);
    env->lazy_fd = INVALID_HANDLE_VALUE;
  }

  if (env->lck_mmap.fd != INVALID_HANDLE_VALUE) {
    (void)osal_closefile(env->lck_mmap.fd);
    env->lck_mmap.fd = INVALID_HANDLE_VALUE;
  }

  if (!resurrect_after_fork) {
    if (env->kvs) {
      for (size_t i = CORE_DBS; i < env->n_dbi; ++i)
        if (env->kvs[i].name.iov_len)
          osal_free(env->kvs[i].name.iov_base);
      osal_free(env->kvs);
      env->n_dbi = CORE_DBS;
      env->kvs = nullptr;
    }
    if (env->page_auxbuf) {
      osal_memalign_free(env->page_auxbuf);
      env->page_auxbuf = nullptr;
    }
    if (env->dbi_seqs) {
      osal_free(env->dbi_seqs);
      env->dbi_seqs = nullptr;
    }
    if (env->dbs_flags) {
      osal_free(env->dbs_flags);
      env->dbs_flags = nullptr;
    }
    if (env->pathname.buffer) {
      osal_free(env->pathname.buffer);
      env->pathname.buffer = nullptr;
    }
    if (env->basal_txn) {
      txn_basal_destroy(env->basal_txn);
      env->basal_txn = nullptr;
    }
  }
  env->stuck_meta = -1;
  return rc;
}

#if MDBX_USE_MINCORE
/*------------------------------------------------------------------------------
 * Проверка размещения/расположения отображенных страниц БД в ОЗУ (mem-in-core),
 * с кешированием этой информации. */

static inline bool bit_tas(uint64_t *field, char bit) {
  const uint64_t m = UINT64_C(1) << bit;
  const bool r = (*field & m) != 0;
  *field |= m;
  return r;
}

static bool mincore_fetch(MDBX_env *const env, const size_t unit_begin) {
  lck_t *const lck = env->lck;
  for (size_t i = 1; i < ARRAY_LENGTH(lck->mincore_cache.begin); ++i) {
    const ptrdiff_t dist = unit_begin - lck->mincore_cache.begin[i];
    if (likely(dist >= 0 && dist < 64)) {
      const pgno_t tmp_begin = lck->mincore_cache.begin[i];
      const uint64_t tmp_mask = lck->mincore_cache.mask[i];
      do {
        lck->mincore_cache.begin[i] = lck->mincore_cache.begin[i - 1];
        lck->mincore_cache.mask[i] = lck->mincore_cache.mask[i - 1];
      } while (--i);
      lck->mincore_cache.begin[0] = tmp_begin;
      lck->mincore_cache.mask[0] = tmp_mask;
      return bit_tas(lck->mincore_cache.mask, (char)dist);
    }
  }

  size_t pages = 64;
  unsigned unit_log = globals.sys_pagesize_ln2;
  unsigned shift = 0;
  if (env->ps > globals.sys_pagesize) {
    unit_log = env->ps2ln;
    shift = env->ps2ln - globals.sys_pagesize_ln2;
    pages <<= shift;
  }

  const size_t offset = unit_begin << unit_log;
  size_t length = pages << globals.sys_pagesize_ln2;
  if (offset + length > env->dxb_mmap.current) {
    length = env->dxb_mmap.current - offset;
    pages = length >> globals.sys_pagesize_ln2;
  }

  if (MDBX_ENABLE_PGOP_STAT)
    env->lck->pgops.mincore.weak += 1;

  uint8_t *const vector = alloca(pages);
  if (unlikely(mincore(ptr_disp(env->dxb_mmap.base, offset), length, (void *)vector))) {
    NOTICE("mincore(+%zu, %zu), err %d", offset, length, errno);
    return false;
  }

  for (size_t i = 1; i < ARRAY_LENGTH(lck->mincore_cache.begin); ++i) {
    lck->mincore_cache.begin[i] = lck->mincore_cache.begin[i - 1];
    lck->mincore_cache.mask[i] = lck->mincore_cache.mask[i - 1];
  }
  lck->mincore_cache.begin[0] = unit_begin;

  uint64_t mask = 0;
#ifdef MINCORE_INCORE
  STATIC_ASSERT(MINCORE_INCORE == 1);
#endif
  for (size_t i = 0; i < pages; ++i) {
    uint64_t bit = (vector[i] & 1) == 0;
    bit <<= i >> shift;
    mask |= bit;
  }

  lck->mincore_cache.mask[0] = ~mask;
  return bit_tas(lck->mincore_cache.mask, 0);
}
#endif /* MDBX_USE_MINCORE */

bool env_is_page_incore(MDBX_env *const env, const pgno_t pgno) {
#if MDBX_USE_MINCORE
  const size_t offset_aligned = floor_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize);
  const unsigned unit_log2 = (env->ps2ln > globals.sys_pagesize_ln2) ? env->ps2ln : globals.sys_pagesize_ln2;
  const size_t unit_begin = offset_aligned >> unit_log2;
  eASSERT0(env, (unit_begin << unit_log2) == offset_aligned);
  const ptrdiff_t dist = unit_begin - env->lck->mincore_cache.begin[0];
  if (likely(dist >= 0 && dist < 64))
    return bit_tas(env->lck->mincore_cache.mask, (char)dist);
  return mincore_fetch(env, unit_begin);
#else
  (void)env;
  (void)pgno;
  return false;
#endif /* MDBX_USE_MINCORE */
}

/*----------------------------------------------------------------------------*/

MDBX_MAYBE_UNUSED __hot static pgno_t *scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) {
  ASSERT(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
  ASSERT(range[-1] == len);
  const pgno_t *const detent = range + len - seq;
  const ptrdiff_t offset = (ptrdiff_t)seq;
  const pgno_t target = (pgno_t)offset;
  if (likely(len > seq + 3)) {
    do {
      const pgno_t diff0 = range[offset + 0] - range[0];
      const pgno_t diff1 = range[offset + 1] - range[1];
      const pgno_t diff2 = range[offset + 2] - range[2];
      const pgno_t diff3 = range[offset + 3] - range[3];
      if (diff0 == target)
        return range + 0;
      if (diff1 == target)
        return range + 1;
      if (diff2 == target)
        return range + 2;
      if (diff3 == target)
        return range + 3;
      range += 4;
    } while (range + 3 < detent);
    if (range == detent)
      return nullptr;
  }
  do
    if (range[offset] - *range == target)
      return range;
  while (++range < detent);
#else
  ASSERT(range[-(ptrdiff_t)len] == len);
  const pgno_t *const detent = range - len + seq;
  const ptrdiff_t offset = -(ptrdiff_t)seq;
  const pgno_t target = (pgno_t)offset;
  if (likely(len > seq + 3)) {
    do {
      const pgno_t diff0 = range[-0] - range[offset - 0];
      const pgno_t diff1 = range[-1] - range[offset - 1];
      const pgno_t diff2 = range[-2] - range[offset - 2];
      const pgno_t diff3 = range[-3] - range[offset - 3];
      /* Смысл вычислений до ветвлений в том, чтобы позволить компилятору
       * загружать и вычислять все значения параллельно. */
      if (diff0 == target)
        return range - 0;
      if (diff1 == target)
        return range - 1;
      if (diff2 == target)
        return range - 2;
      if (diff3 == target)
        return range - 3;
      range -= 4;
    } while (range > detent + 3);
    if (range == detent)
      return nullptr;
  }
  do
    if (*range - range[offset] == target)
      return range;
  while (--range > detent);
#endif /* pnl_t sort-order */
  return nullptr;
}

MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl, const size_t seq) {
  size_t begin = MDBX_PNL_ASCENDING ? 1 : pnl_size(pnl);
#if MDBX_PNL_ASCENDING
  while (seq <= pnl_size(pnl) - begin) {
    if (pnl[begin + seq] - pnl[begin] == seq)
      return pnl + begin;
    ++begin;
  }
#else
  while (begin > seq) {
    if (pnl[begin - seq] - pnl[begin] == seq)
      return pnl + begin;
    --begin;
  }
#endif /* pnl_t sort-order */
  return nullptr;
}

#if defined(_MSC_VER) && !defined(__builtin_clz) && !__has_builtin(__builtin_clz)
MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clz(uint32_t value) {
  unsigned long index;
  _BitScanReverse(&index, value);
  return 31 - index;
}
#endif /* _MSC_VER */

#if defined(_MSC_VER) && !defined(__builtin_clzl) && !__has_builtin(__builtin_clzl)
MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) {
  unsigned long index;
#ifdef _WIN64
  ASSERT(sizeof(value) == 8);
  _BitScanReverse64(&index, value);
  return 63 - index;
#else
  ASSERT(sizeof(value) == 4);
  _BitScanReverse(&index, value);
  return 31 - index;
#endif
}
#endif /* _MSC_VER */

#if !MDBX_PNL_ASCENDING

#if !defined(MDBX_ATTRIBUTE_TARGET) && (__has_attribute(__target__) || __GNUC_PREREQ(5, 0))
#define MDBX_ATTRIBUTE_TARGET(target) __attribute__((__target__(target)))
#endif /* MDBX_ATTRIBUTE_TARGET */

#ifndef MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND
/* Workaround for GCC's bug with `-m32 -march=i686 -Ofast`
 * gcc/i686-buildroot-linux-gnu/12.2.0/include/xmmintrin.h:814:1:
 *     error: inlining failed in call to 'always_inline' '_mm_movemask_ps':
 *            target specific option mismatch */
#if !defined(__FAST_MATH__) || !__FAST_MATH__ || !defined(__GNUC__) || defined(__e2k__) || defined(__clang__) ||       \
    defined(__amd64__) || defined(__SSE2__)
#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 0
#else
#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 1
#endif
#endif /* MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND */

#if defined(__SSE2__) && defined(__SSE__)
#define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */
#elif (defined(_M_IX86_FP) && _M_IX86_FP >= 2) || defined(__amd64__)
#define __SSE2__
#define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND
#define MDBX_ATTRIBUTE_TARGET_SSE2 MDBX_ATTRIBUTE_TARGET("sse,sse2")
#endif /* __SSE2__ */

#if defined(__AVX2__)
#define MDBX_ATTRIBUTE_TARGET_AVX2 /* nope */
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND
#define MDBX_ATTRIBUTE_TARGET_AVX2 MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2")
#endif /* __AVX2__ */

#if defined(MDBX_ATTRIBUTE_TARGET_AVX2)
#if defined(__AVX512BW__)
#define MDBX_ATTRIBUTE_TARGET_AVX512BW /* nope */
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND &&                \
    (__GNUC_PREREQ(6, 0) || __CLANG_PREREQ(5, 0))
#define MDBX_ATTRIBUTE_TARGET_AVX512BW MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2,avx512bw")
#endif /* __AVX512BW__ */
#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 for MDBX_ATTRIBUTE_TARGET_AVX512BW */

#ifdef MDBX_ATTRIBUTE_TARGET_SSE2
MDBX_ATTRIBUTE_TARGET_SSE2 static __always_inline unsigned
diffcmp2mask_sse2(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) {
  const __m128i f = _mm_loadu_si128((const __m128i *)ptr);
  const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset));
  const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern);
  return _mm_movemask_ps(*(const __m128 *)&cmp);
}

MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_SSE2 static pgno_t *scan4seq_sse2(pgno_t *range, const size_t len,
                                                                                const size_t seq) {
  ASSERT(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Not implemented"
#endif /* MDBX_PNL_ASCENDING */
  ASSERT(range[-(ptrdiff_t)len] == len);
  pgno_t *const detent = range - len + seq;
  const ptrdiff_t offset = -(ptrdiff_t)seq;
  const pgno_t target = (pgno_t)offset;
  const __m128i pattern = _mm_set1_epi32(target);
  uint8_t mask;
  if (likely(len > seq + 3)) {
    do {
      mask = (uint8_t)diffcmp2mask_sse2(range - 3, offset, pattern);
      if (mask) {
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
      found:
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
        return range + 28 - __builtin_clz(mask);
      }
      range -= 4;
    } while (range > detent + 3);
    if (range == detent)
      return nullptr;
  }

  /* Далее происходит чтение от 4 до 12 лишних байт, которые могут быть не
   * только за пределами региона выделенного под PNL, но и пересекать границу
   * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению.
   * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
  const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */;
  if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
    const unsigned extra = (unsigned)(detent + 4 - range);
    ASSERT(extra > 0 && extra < 4);
    mask = 0xF << extra;
    mask &= diffcmp2mask_sse2(range - 3, offset, pattern);
    if (mask)
      goto found;
    return nullptr;
  }
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
  do
    if (*range - range[offset] == target)
      return range;
  while (--range != detent);
  return nullptr;
}
#endif /* MDBX_ATTRIBUTE_TARGET_SSE2 */

#ifdef MDBX_ATTRIBUTE_TARGET_AVX2
MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned
diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset, const __m256i pattern) {
  const __m256i f = _mm256_loadu_si256((const __m256i *)ptr);
  const __m256i l = _mm256_loadu_si256((const __m256i *)(ptr + offset));
  const __m256i cmp = _mm256_cmpeq_epi32(_mm256_sub_epi32(f, l), pattern);
  return _mm256_movemask_ps(*(const __m256 *)&cmp);
}

MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned
diffcmp2mask_sse2avx(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) {
  const __m128i f = _mm_loadu_si128((const __m128i *)ptr);
  const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset));
  const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern);
  return _mm_movemask_ps(*(const __m128 *)&cmp);
}

MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX2 static pgno_t *scan4seq_avx2(pgno_t *range, const size_t len,
                                                                                const size_t seq) {
  ASSERT(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Since 2026-04-01 alternatives to MDBX_PNL_ASCENDING = 0 are no longer supported."
#endif /* MDBX_PNL_ASCENDING */
  ASSERT(range[-(ptrdiff_t)len] == len);
  pgno_t *const detent = range - len + seq;
  const ptrdiff_t offset = -(ptrdiff_t)seq;
  const pgno_t target = (pgno_t)offset;
  const __m256i pattern = _mm256_set1_epi32(target);
  uint8_t mask;
  if (likely(len > seq + 7)) {
    do {
      mask = (uint8_t)diffcmp2mask_avx2(range - 7, offset, pattern);
      if (mask) {
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
      found:
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
        return range + 24 - __builtin_clz(mask);
      }
      range -= 8;
    } while (range > detent + 7);
    if (range == detent)
      return nullptr;
  }

  /* Далее происходит чтение от 4 до 28 лишних байт, которые могут быть не
   * только за пределами региона выделенного под PNL, но и пересекать границу
   * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению.
   * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
  const unsigned on_page_safe_mask = 0xfe0 /* enough for '-31' bytes offset */;
  if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
    const unsigned extra = (unsigned)(detent + 8 - range);
    ASSERT(extra > 0 && extra < 8);
    mask = 0xFF << extra;
    mask &= diffcmp2mask_avx2(range - 7, offset, pattern);
    if (mask)
      goto found;
    return nullptr;
  }
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
  if (range - 3 > detent) {
    mask = diffcmp2mask_sse2avx(range - 3, offset, *(const __m128i *)&pattern);
    if (mask)
      return range + 28 - __builtin_clz(mask);
    range -= 4;
  }
  while (range > detent) {
    if (*range - range[offset] == target)
      return range;
    --range;
  }
  return nullptr;
}
#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 */

#ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW
MDBX_ATTRIBUTE_TARGET_AVX512BW static __always_inline unsigned
diffcmp2mask_avx512bw(const pgno_t *const ptr, const ptrdiff_t offset, const __m512i pattern) {
  const __m512i f = _mm512_loadu_si512((const __m512i *)ptr);
  const __m512i l = _mm512_loadu_si512((const __m512i *)(ptr + offset));
  return _mm512_cmpeq_epi32_mask(_mm512_sub_epi32(f, l), pattern);
}

MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX512BW static pgno_t *scan4seq_avx512bw(pgno_t *range, const size_t len,
                                                                                        const size_t seq) {
  ASSERT(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Since 2026-04-01 alternatives to MDBX_PNL_ASCENDING = 0 are no longer supported."
#endif /* MDBX_PNL_ASCENDING */
  ASSERT(range[-(ptrdiff_t)len] == len);
  pgno_t *const detent = range - len + seq;
  const ptrdiff_t offset = -(ptrdiff_t)seq;
  const pgno_t target = (pgno_t)offset;
  const __m512i pattern = _mm512_set1_epi32(target);
  unsigned mask;
  if (likely(len > seq + 15)) {
    do {
      mask = diffcmp2mask_avx512bw(range - 15, offset, pattern);
      if (mask) {
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
      found:
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
        return range + 16 - __builtin_clz(mask);
      }
      range -= 16;
    } while (range > detent + 15);
    if (range == detent)
      return nullptr;
  }

  /* Далее происходит чтение от 4 до 60 лишних байт, которые могут быть не
   * только за пределами региона выделенного под PNL, но и пересекать границу
   * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению.
   * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
  const unsigned on_page_safe_mask = 0xfc0 /* enough for '-63' bytes offset */;
  if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
    const unsigned extra = (unsigned)(detent + 16 - range);
    ASSERT(extra > 0 && extra < 16);
    mask = 0xFFFF << extra;
    mask &= diffcmp2mask_avx512bw(range - 15, offset, pattern);
    if (mask)
      goto found;
    return nullptr;
  }
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
  if (range - 7 > detent) {
    mask = diffcmp2mask_avx2(range - 7, offset, *(const __m256i *)&pattern);
    if (mask)
      return range + 24 - __builtin_clz(mask);
    range -= 8;
  }
  if (range - 3 > detent) {
    mask = diffcmp2mask_sse2avx(range - 3, offset, *(const __m128i *)&pattern);
    if (mask)
      return range + 28 - __builtin_clz(mask);
    range -= 4;
  }
  while (range > detent) {
    if (*range - range[offset] == target)
      return range;
    --range;
  }
  return nullptr;
}
#endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */

#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr, const ptrdiff_t offset,
                                                const uint32x4_t pattern) {
  const uint32x4_t f = vld1q_u32(ptr);
  const uint32x4_t l = vld1q_u32(ptr + offset);
  const uint16x4_t cmp = vmovn_u32(vceqq_u32(vsubq_u32(f, l), pattern));
  if (sizeof(size_t) > 7)
    return vget_lane_u64(vreinterpret_u64_u16(cmp), 0);
  else
    return vget_lane_u32(vreinterpret_u32_u8(vmovn_u16(vcombine_u16(cmp, cmp))), 0);
}

__hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, const size_t seq) {
  ASSERT(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Since 2026-04-01 alternatives to MDBX_PNL_ASCENDING = 0 are no longer supported."
#endif /* MDBX_PNL_ASCENDING */
  ASSERT(range[-(ptrdiff_t)len] == len);
  pgno_t *const detent = range - len + seq;
  const ptrdiff_t offset = -(ptrdiff_t)seq;
  const pgno_t target = (pgno_t)offset;
  const uint32x4_t pattern = vmovq_n_u32(target);
  size_t mask;
  if (likely(len > seq + 3)) {
    do {
      mask = diffcmp2mask_neon(range - 3, offset, pattern);
      if (mask) {
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
      found:
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
        return ptr_disp(range, -(__builtin_clzl(mask) >> sizeof(size_t) / 4));
      }
      range -= 4;
    } while (range > detent + 3);
    if (range == detent)
      return nullptr;
  }

  /* Далее происходит чтение от 4 до 12 лишних байт, которые могут быть не
   * только за пределами региона выделенного под PNL, но и пересекать границу
   * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению.
   * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
  const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */;
  if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
    const unsigned extra = (unsigned)(detent + 4 - range);
    ASSERT(extra > 0 && extra < 4);
    mask = (~(size_t)0) << (extra * sizeof(size_t) * 2);
    mask &= diffcmp2mask_neon(range - 3, offset, pattern);
    if (mask)
      goto found;
    return nullptr;
  }
#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */
  do
    if (*range - range[offset] == target)
      return range;
  while (--range != detent);
  return nullptr;
}
#endif /* __ARM_NEON || __ARM_NEON__ */

#if defined(__AVX512BW__) && defined(MDBX_ATTRIBUTE_TARGET_AVX512BW)
#define scan4seq_default scan4seq_avx512bw
#define scan4seq_impl scan4seq_default
#elif defined(__AVX2__) && defined(MDBX_ATTRIBUTE_TARGET_AVX2)
#define scan4seq_default scan4seq_avx2
#elif defined(__SSE2__) && defined(MDBX_ATTRIBUTE_TARGET_SSE2)
#define scan4seq_default scan4seq_sse2
#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define scan4seq_default scan4seq_neon
/* Choosing of another variants should be added here. */
#endif /* scan4seq_default */

#endif /* MDBX_PNL_ASCENDING */

#ifndef scan4seq_default
#define scan4seq_default scan4seq_fallback
#endif /* scan4seq_default */

#ifdef scan4seq_impl
/* The scan4seq_impl() is the best or no alternatives */
#elif !MDBX_HAVE_BUILTIN_CPU_SUPPORTS
/* The scan4seq_default() will be used since no cpu-features detection support
 * from compiler. Please don't ask to implement cpuid-based detection and don't
 * make such PRs. */
#define scan4seq_impl scan4seq_default
#else
/* Selecting the most appropriate implementation at runtime,
 * depending on the available CPU features. */
static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq);
static pgno_t *(*scan4seq_impl)(pgno_t *range, const size_t len, const size_t seq) = scan4seq_resolver;

static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq) {
  pgno_t *(*choice)(pgno_t *range, const size_t len, const size_t seq) = nullptr;
#if __has_builtin(__builtin_cpu_init) || defined(__BUILTIN_CPU_INIT__) || __GNUC_PREREQ(4, 8)
  __builtin_cpu_init();
#endif /* __builtin_cpu_init() */
#ifdef MDBX_ATTRIBUTE_TARGET_SSE2
  if (__builtin_cpu_supports("sse2"))
    choice = scan4seq_sse2;
#endif /* MDBX_ATTRIBUTE_TARGET_SSE2 */
#ifdef MDBX_ATTRIBUTE_TARGET_AVX2
  if (__builtin_cpu_supports("avx2"))
    choice = scan4seq_avx2;
#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 */
#ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW
  if (__builtin_cpu_supports("avx512bw"))
    choice = scan4seq_avx512bw;
#endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */
  /* Choosing of another variants should be added here. */
  scan4seq_impl = choice ? choice : scan4seq_default;
  return scan4seq_impl(range, len, seq);
}
#endif /* scan4seq_impl */

/*----------------------------------------------------------------------------*/

static inline bool is_reclaimable(MDBX_txn *txn, const MDBX_cursor *mc, const uint8_t flags) {
  /* If txn is updating the GC, then the retired-list cannot play catch-up with
   * itself by growing while trying to save it. */
  STATIC_ASSERT(ALLOC_RESERVE == z_gcu_preparation);
  if (mc->tree == &txn->dbs[FREE_DBI] && !((flags | mc->flags) & z_gcu_preparation))
    return false;

  /* avoid search inside empty tree and while tree is updating,
     https://libmdbx.dqdkfa.ru/dead-github/issues/31 */
  if (unlikely(txn->dbs[FREE_DBI].items == 0)) {
    txn->flags |= txn_gc_drained;
    return false;
  }

  return true;
}

__hot static pgno_t repnl_get_single(MDBX_txn *txn) {
  const size_t len = pnl_size(txn->wr.repnl);
  ASSERT(len > 0);
  pgno_t *target = MDBX_PNL_EDGE(txn->wr.repnl);
  const ptrdiff_t dir = MDBX_PNL_ASCENDING ? 1 : -1;

  /* Есть ТРИ потенциально выигрышные, но противо-направленные тактики:
   *
   * 1. Стараться использовать страницы с наименьшими номерами. Так обмен с
   * диском будет более кучным, а у страниц ближе к концу БД будет больше шансов
   * попасть под авто-компактификацию. Частично эта тактика уже реализована, но
   * для её эффективности требуется явно приоритезировать выделение страниц:
   *   - поддерживать два repnl, для ближних и для дальних страниц;
   *   - использовать страницы из дальнего списка, если первый пуст,
   *     а второй слишком большой, либо при пустой GC.
   *
   * 2. Стараться выделять страницы последовательно. Так записываемые на диск
   * регионы будут линейными, что принципиально ускоряет запись на HDD.
   * Одновременно, в среднем это не повлияет на чтение, точнее говоря, если
   * порядок чтения не совпадает с порядком изменения (иначе говоря, если
   * чтение не коррелирует с обновлениями и/или вставками) то не повлияет, иначе
   * может ускорить. Однако, последовательности в среднем достаточно редки.
   * Поэтому для эффективности требуется аккумулировать и поддерживать в ОЗУ
   * огромные списки страниц, а затем сохранять их обратно в БД. Текущий формат
   * БД (без сжатых битовых карт) для этого крайне не удачен. Поэтому эта тактика не
   * имеет шансов быть успешной без смены формата БД (Mithril).
   *
   * 3. Стараться экономить последовательности страниц. Это позволяет избегать
   * лишнего чтения/поиска в GC при более-менее постоянном размещении и/или
   * обновлении данных требующих более одной страницы. Проблема в том, что без
   * информации от приложения библиотека не может знать насколько
   * востребованными будут последовательности в ближайшей перспективе, а
   * экономия последовательностей "на всякий случай" не только затратна
   * сама-по-себе, но и работает во вред (добавляет хаоса).
   *
   * Поэтому:
   *  - в TODO добавляется разделение repnl на «ближние» и «дальние» страницы,
   *    с последующей реализацией первой тактики;
   *  - преимущественное использование последовательностей отправляется
   *    в MithrilDB как составляющая "HDD frendly" feature;
   *  - реализованная в 3757eb72f7c6b46862f8f17881ac88e8cecc1979 экономия
   *    последовательностей отключается через MDBX_ENABLE_SAVING_SEQUENCES=0.
   *
   * В качестве альтернативы для безусловной «экономии» последовательностей,
   * в следующих версиях libmdbx, вероятно, будет предложено
   * API для взаимодействия с GC:
   *  - получение размера GC, включая гистограммы размеров последовательностей
   *    и близости к концу БД;
   *  - включение формирования "линейного запаса" для последующего использования
   *    в рамках текущей транзакции;
   *  - намеренная загрузка GC в память для коагуляции и "выпрямления";
   *  - намеренное копирование данных из страниц в конце БД для последующего
   *    из освобождения, т.е. контролируемая компактификация по запросу. */

#ifndef MDBX_ENABLE_SAVING_SEQUENCES
#define MDBX_ENABLE_SAVING_SEQUENCES 0
#endif
  if (MDBX_ENABLE_SAVING_SEQUENCES && unlikely(target[dir] == *target + 1) && len > 2) {
    /* Пытаемся пропускать последовательности при наличии одиночных элементов.
     * TODO: необходимо кэшировать пропускаемые последовательности
     * чтобы не сканировать список сначала при каждом выделении. */
    pgno_t *scan = target + dir + dir;
    size_t left = len;
    do {
      if (likely(scan[-dir] != *scan - 1 && *scan + 1 != scan[dir])) {
#if MDBX_PNL_ASCENDING
        target = scan;
        break;
#else
        /* вырезаем элемент с перемещением хвоста */
        const pgno_t pgno = *scan;
        pnl_setsize(txn->wr.repnl, len - 1);
        while (++scan <= target)
          scan[-1] = *scan;
        return pgno;
#endif
      }
      scan += dir;
    } while (--left > 2);
  }

  const pgno_t pgno = *target;
#if MDBX_PNL_ASCENDING
  /* вырезаем элемент с перемещением хвоста */
  pnl_setsize(txn->wr.repnl, len - 1);
  for (const pgno_t *const end = txn->wr.repnl + len - 1; target <= end; ++target)
    *target = target[1];
#else
  /* перемещать хвост не нужно, просто усекам список */
  pnl_setsize(txn->wr.repnl, len - 1);
#endif /* MDBX_PNL_ASCENDING */
  return pgno;
}

__hot pgno_t gc_repnl_get_single(MDBX_txn *txn) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  return pnl_size(txn->wr.repnl) ? repnl_get_single(txn) : 0;
}

__hot pgno_t gc_repnl_get_sequence(MDBX_txn *txn, const size_t num, uint8_t flags) {
  const size_t len = pnl_size(txn->wr.repnl);
  pgno_t *edge = MDBX_PNL_EDGE(txn->wr.repnl);
  ASSERT(len >= num && num > 1);
  const size_t seq = num - 1;
#if !MDBX_PNL_ASCENDING
  if (edge[-(ptrdiff_t)seq] - *edge == seq &&
      (likely((flags & ALLOC_EXACTLY) == 0) || len == num || edge[-(ptrdiff_t)num] - *edge != num)) {
    if (likely((flags & ALLOC_RESERVE) == 0)) {
      ASSERT(edge == scan4range_checker(txn->wr.repnl, seq));
      /* перемещать хвост не нужно, просто усекам список */
      pnl_setsize(txn->wr.repnl, len - num);
    }
    return *edge;
  }
#endif /* !MDBX_PNL_ASCENDING */
  pgno_t *target = scan4seq_impl(edge, len, seq);
  ASSERT(target == scan4range_checker(txn->wr.repnl, seq));
  while (target) {
    if (unlikely(flags & ALLOC_EXACTLY) && len > num) {
      const ptrdiff_t step = MDBX_PNL_ASCENDING ? -1 : 1;
      if (target[step] - target[0] == 1) {
        /* последовательность продолжается, пропускаем её */
        size_t left =
            MDBX_PNL_ASCENDING ? target - &MDBX_PNL_FIRST(txn->wr.repnl) : &MDBX_PNL_LAST(txn->wr.repnl) - target;
        do {
          target += step;
          if (--left < num)
            /* осталось меньше чем нужно, последовательности целевой длины быть не может */
            return 0;
        } while (target[step] - target[0] == 1);
        /* продолжаем поиск дальше */
        target = scan4seq_impl(target, left, seq);
        continue;
      }
      /* найденая последовательность ровно необхожимой длины */
    }
    const pgno_t pgno = *target;
    if (likely((flags & ALLOC_RESERVE) == 0)) {
      /* вырезаем найденную последовательность с перемещением хвоста */
      pnl_setsize(txn->wr.repnl, len - num);
#if MDBX_PNL_ASCENDING
      for (const pgno_t *const end = txn->wr.repnl + len - num; target <= end; ++target)
        *target = target[num];
#else
      for (const pgno_t *const end = txn->wr.repnl + len; ++target <= end;)
        target[-(ptrdiff_t)num] = *target;
#endif /* MDBX_PNL_ASCENDING */
    }
    return pgno;
  }
  return 0;
}

bool gc_repnl_has_span(const MDBX_txn *txn, const size_t num) {
  return (num > 1) ? gc_repnl_get_sequence((MDBX_txn *)txn, num, ALLOC_RESERVE) != 0
                   : !MDBX_PNL_IS_EMPTY(txn->wr.repnl);
}

static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn, const MDBX_cursor *const mc,
                                        const pgno_t pgno, const size_t num) {
#if MDBX_ENABLE_PROFGC
  size_t majflt_before;
  const uint64_t cputime_before = osal_cputime(&majflt_before);
  gc_prof_stat_t *const prof =
      (cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work;
#else
  (void)mc;
#endif /* MDBX_ENABLE_PROFGC */
  ENSURE_OBJ(env, pgno >= NUM_METAS);

  pgr_t ret;
  bool need_clean = (env->flags & MDBX_PAGEPERTURB) != 0;
  if (env->flags & MDBX_WRITEMAP) {
    ret.page = pgno2page(env, pgno);
    MDBX_ASAN_UNPOISON_MEMORY_REGION(ret.page, pgno2bytes(env, num));
    VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num));

    /* Содержимое выделенной страницы не нужно, но если страница отсутствует
     * в ОЗУ (что весьма вероятно), то любое обращение к ней приведет
     * к page-fault:
     *  - прерыванию по отсутствию страницы;
     *  - переключение контекста в режим ядра с засыпанием процесса;
     *  - чтение страницы с диска;
     *  - обновление PTE и пробуждением процесса;
     *  - переключение контекста по доступности ЦПУ.
     *
     * Пытаемся минимизировать накладные расходы записывая страницу, что при
     * наличии unified page cache приведет к появлению страницы в ОЗУ без чтения
     * с диска. При этом запись на диск должна быть отложена адекватным ядром,
     * так как страница отображена в память в режиме чтения-записи и следом в
     * неё пишет ЦПУ. */

    /* В случае если страница в памяти процесса, то излишняя запись может быть
     * достаточно дорогой. Кроме системного вызова и копирования данных, в особо
     * одаренных ОС при этом могут включаться файловая система, выделяться
     * временная страница, пополняться очереди асинхронного выполнения,
     * обновляться PTE с последующей генерацией page-fault и чтением данных из
     * грязной I/O очереди. Из-за этого штраф за лишнюю запись может быть
     * сравним с избегаемым ненужным чтением. */
    if (txn->wr.prefault_write_activated) {
      void *const pattern = ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2);
      size_t file_offset = pgno2bytes(env, pgno);
      if (likely(num == 1)) {
        if (!env_is_page_incore(env, pgno)) {
          osal_pwrite(env->lazy_fd, pattern, env->ps, file_offset);
          if (MDBX_ENABLE_PGOP_STAT)
            env->lck->pgops.prefault.weak += 1;
          need_clean = false;
        }
      } else {
        struct iovec iov[MDBX_AUXILARY_IOV_MAX];
        size_t n = 0, cleared = 0;
        for (size_t i = 0; i < num; ++i) {
          if (!env_is_page_incore(env, pgno + (pgno_t)i)) {
            ++cleared;
            iov[n].iov_len = env->ps;
            iov[n].iov_base = pattern;
            if (unlikely(++n == MDBX_AUXILARY_IOV_MAX)) {
              osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, file_offset);
              if (MDBX_ENABLE_PGOP_STAT)
                env->lck->pgops.prefault.weak += 1;
              file_offset += pgno2bytes(env, MDBX_AUXILARY_IOV_MAX);
              n = 0;
            }
          }
        }
        if (likely(n > 0)) {
          osal_pwritev(env->lazy_fd, iov, n, file_offset);
          if (MDBX_ENABLE_PGOP_STAT)
            env->lck->pgops.prefault.weak += 1;
        }
        if (cleared == num)
          need_clean = false;
      }
    }
  } else {
    ret.page = page_shadow_alloc(txn, num);
    if (unlikely(!ret.page)) {
      ret.err = MDBX_ENOMEM;
      goto bailout;
    }
  }

  if (unlikely(need_clean))
    memset(ret.page, -1, pgno2bytes(env, num));

  VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num));
  ret.page->pgno = pgno;
  ret.page->dupfix_ksize = 0;
  ret.page->flags = 0;
  if (num > 1 && CHECKS0_ENABLED()) {
    ret.page->pages = (pgno_t)num;
    ret.page->flags = P_LARGE;
  }

  ret.err = page_dirty(txn, ret.page, (pgno_t)num);
bailout:
  tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated));
#if MDBX_ENABLE_PROFGC
  size_t majflt_after;
  prof->xtime_cpu += osal_cputime(&majflt_after) - cputime_before;
  prof->majflt += (uint32_t)(majflt_after - majflt_before);
#endif /* MDBX_ENABLE_PROFGC */
  return ret;
}

static txnid_t shapshot_oldest_force_rescan(MDBX_txn *const txn) {
  atomic_store32(&txn->env->lck->rdt_refresh_flag, true, mo_AcquireRelease);
  return mvcc_shapshot_oldest_rw(txn).oldest_txnid;
}

const char *gc_check_keylen(size_t const key_len) {
  return likely(key_len == sizeof(txnid_t)) ? nullptr : "invalid GC key-length";
}

const char *gc_check_rowdata(const MDBX_txn *const txn, const MDBX_val data) {
  if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < sizeof(pgno_t)))
    return "invalid length of GC-record";
  if (unlikely((size_t)data.iov_base % sizeof(pgno_t)))
    return "invalid alignment of GC-record";

  const const_pnl_t pnl = data.iov_base;
  if (unlikely(data.iov_len < MDBX_PNL_SIZEOF(pnl)))
    return "trimmed GC-record";
  if (unlikely(!pnl_check(pnl, txn->geo.first_unallocated)))
    return "wrong GC-record";

  return /* no error */ nullptr;
}

glr_t gc_row_pnl(const MDBX_txn *const txn, const MDBX_val data) {
  glr_t result = {.pnl = (const pgno_t *)data.iov_base, .err = MDBX_SUCCESS, .reason = gc_check_rowdata(txn, data)};
  if (unlikely(result.reason)) {
    ERROR("%s/%d: %s", "MDBX_CORRUPTED", result.err = MDBX_CORRUPTED, result.reason);
    result.pnl = nullptr;
  }
  return result;
}

MDBX_cursor *gc_cursor_init(MDBX_txn *txn) {
  MDBX_cursor *gc = ptr_disp(txn->env->basal_txn, sizeof(MDBX_txn));
  gc->txn = txn;
  gc->tree = txn->dbs;
  gc->dbi_state = txn->dbi_state;
  gc->top_and_flags = z_fresh_mark;
  return gc;
}

pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) {
  pgr_t ret;
  MDBX_txn *const txn = mc->txn;
  MDBX_env *const env = txn->env;
#if MDBX_ENABLE_PROFGC
  gc_prof_stat_t *const prof =
      (cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work;
  prof->spe_counter += 1;
#endif /* MDBX_ENABLE_PROFGC */

  /* Если взведен флажок ALLOC_RESERVE, то требуется только обеспечение соответствующего резерва в txn->wr.repnl
   * и/или txn->wr.gc.reclaimed, но без выделения и возврата страницы. При этом возможны три варианта вызова:
   *   1. num == 0 — требуется слот для возврата в GC остатков ранее переработанных/извлеченных страниц,
   *      при этом нет смысла перерабатывать длинные записи, так как тогда дефицит свободных id/слотов не уменьшится;
   *   2. num == 1 — требуется увеличение резерва перед обновлением GC;
   *   3. num > 1 — требуется последовательность страниц для сохранения retired-страниц
   *      при выключенном MDBX_ENABLE_BIGFOOT. */
  eASSERT0(env, num > 0 || (flags & ALLOC_RESERVE));
  eASSERT1(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated));

  size_t newnext;
  const uint64_t monotime_begin = (MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit)) ? osal_monotime() : 0;
  struct monotime_cache now_cache;
  now_cache.expire_countdown = 1 /* старт с 1 позволяет избавиться как от лишних системных вызовов когда
                                    лимит времени задан нулевой или уже исчерпан, так и от подсчета
                                    времени при не-достижении rp_augment_limit */
      ;
  now_cache.value = monotime_begin;
  pgno_t pgno = 0;
  if (num > 1) {
#if MDBX_ENABLE_PROFGC
    prof->xpages += 1;
#endif /* MDBX_ENABLE_PROFGC */
    if (pnl_size(txn->wr.repnl) >= num) {
      eASSERT0(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated &&
                        MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated);
      pgno = gc_repnl_get_sequence(txn, num, flags);
      if (likely(pgno))
        goto done;
    }
  } else {
    eASSERT0(env, num == 0 || pnl_size(txn->wr.repnl) == 0 || (flags & ALLOC_RESERVE));
  }

  //---------------------------------------------------------------------------

  if (unlikely(!is_reclaimable(txn, mc, flags))) {
    eASSERT0(env, (txn->flags & txn_gc_drained) || num > 1 || mc->tree == &txn->dbs[FREE_DBI]);
    goto no_gc;
  }

  eASSERT0(env, (flags & (ALLOC_COALESCE | ALLOC_LIFO | ALLOC_SHOULD_SCAN)) == 0);
  flags += (env->flags & MDBX_LIFORECLAIM) ? ALLOC_LIFO : 0;

  /* Не коагулируем записи в случае запроса слота для возврата страниц в GC. Иначе попытка увеличить резерв
   * может приводить к необходимости ещё большего резерва из-за увеличения списка переработанных страниц. */
  if (num > 0 && txn->dbs[FREE_DBI].branch_pages && pnl_size(txn->wr.repnl) < env->maxgc_large1page / 2)
    flags += ALLOC_COALESCE;

  txn->wr.prefault_write_activated = !env->incore && env->options.prefault_write;
  if (txn->wr.prefault_write_activated) {
    /* Проверка посредством minicore() существенно снижает затраты, но в
     * простейших случаях (тривиальный бенчмарк) интегральная производительность
     * становится вдвое меньше. А на платформах без mincore() и с проблемной
     * подсистемой виртуальной памяти ситуация может быть многократно хуже.
     * Поэтому избегаем затрат в ситуациях когда prefault-write скорее всего не
     * нужна. */
    const bool readahead_enabled = env->lck->readahead_anchor & 1;
    const pgno_t readahead_edge = env->lck->readahead_anchor >> 1;
    if (/* Не суетимся если GC почти пустая и БД маленькая */
        (txn->dbs[FREE_DBI].branch_pages == 0 && txn->geo.now < 1234) ||
        /* Не суетимся если страница в зоне включенного упреждающего чтения */
        (readahead_enabled && pgno + num < readahead_edge))
      txn->wr.prefault_write_activated = false;
  }

  MDBX_cursor *const gc = gc_cursor_init(txn);
retry_gc_refresh_detent:
  txn_gc_detent(txn);
retry_gc_have_detent:
  if (unlikely(txn->env->gc.detent > txn_basis_snapshot(txn))) {
    FATAL("unexpected/invalid gc-detent %" PRIaTXN " for current-txnid %" PRIaTXN, txn->env->gc.detent, txn->txnid);
    ret.err = MDBX_PROBLEM;
    goto fail;
  }

  txnid_t id = 0;
  MDBX_cursor_op op = MDBX_FIRST;
  if (flags & ALLOC_LIFO) {
    /* Begin lookup backward from oldest reader */
    id = txn->env->gc.detent + 1;
    op = MDBX_SET_RANGE;
  } else {
    /* Continue lookup forward from last-reclaimed */
    id = rkl_highest(&txn->wr.gc.reclaimed);
    if (id) {
      id += 1;
      op = MDBX_SET_RANGE;
      if (id > txn->env->gc.detent)
        goto depleted_gc;
    }
  }

next_gc:
#if MDBX_ENABLE_PROFGC
  prof->rsteps += 1
#endif /* MDBX_ENABLE_PROFGC */
      ;
#ifndef _MSC_VER
  MDBX_val key = {.iov_base = &id, .iov_len = sizeof(id)};
#else
  /* avoid MSVC crash and/or ICE */
  MDBX_val key;
  key.iov_base = &id;
  key.iov_len = sizeof(id);
#endif /* _MSC_VER */

  /* Seek first/next GC record */
  ret.err = cursor_ops(gc, &key, nullptr, op);
  if (unlikely(ret.err != MDBX_SUCCESS)) {
    if (unlikely(ret.err != MDBX_NOTFOUND))
      goto fail;
    if ((flags & ALLOC_LIFO) && op == MDBX_SET_RANGE) {
      op = MDBX_PREV;
      goto next_gc;
    }
    goto depleted_gc;
  }
  if (unlikely(gc_check_keylen(key.iov_len))) {
    ERROR("%s/%d: %s", "corrupted GC-record", ret.err = MDBX_CORRUPTED, gc_check_keylen(key.iov_len));
    goto fail;
  }

  id = unaligned_peek_u64(4, key.iov_base);
  if (flags & ALLOC_LIFO) {
    op = MDBX_PREV;
    if (id > txn->env->gc.detent || gc_is_reclaimed(txn, id))
      goto next_gc;
  } else {
    if (unlikely(id > txn->env->gc.detent))
      goto depleted_gc;
    op = MDBX_NEXT;
    if (gc_is_reclaimed(txn, id))
      goto next_gc;
  }
  txn->flags &= ~txn_gc_drained;

  /* Reading GC record */
  MDBX_val data;
  page_t *const mp = gc->pg[gc->top];
  if (unlikely((ret.err = node_read(gc, page_node(mp, gc->ki[gc->top]), &data, mp)) != MDBX_SUCCESS))
    goto fail;

  const glr_t glr = gc_row_pnl(txn, data);
  if (unlikely(glr.err != MDBX_SUCCESS)) {
    ERROR("%s/%d: %s", "corrupted GC-record", ret.err = glr.err, glr.reason);
    goto fail;
  }

  const pgno_t *const gc_pnl = glr.pnl;
  const size_t gc_len = pnl_size(gc_pnl);
  TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + pnl_size(txn->wr.repnl));

  if (unlikely(!num)) {
    /* TODO: Проверка критериев пункта 2 сформулированного в gc_provide_slots().
     * Сейчас тут сильно упрощенная и не совсем верная проверка, так как пока недоступна информация о кол-ве имеющихся
     * слотов и их дефиците для возврата wr.repl. */
    if (gc_len > env->maxgc_large1page / 4 * 3
        /* если запись достаточно длинная, то переработка слота не особо увеличит место для возврата wr.repl, и т.п. */
        && pnl_size(txn->wr.repnl) + gc_len > env->maxgc_large1page /* не помещается в хвост */) {
      DEBUG("avoid reclaiming %" PRIaTXN " slot, since it is too long (%zu)", id, gc_len);
      ret.err = MDBX_NOTFOUND;
      goto reserve_done;
    }
  }

  if (unlikely(gc_len + pnl_size(txn->wr.repnl) /* Don't try to coalesce too much. */ >= env->maxgc_large1page)) {
    if (flags & ALLOC_SHOULD_SCAN) {
      eASSERT0(env, (flags & ALLOC_COALESCE) /* && !(flags & ALLOC_RESERVE) */ && num > 0);
#if MDBX_ENABLE_PROFGC
      env->lck->pgops.gc_prof.coalescences += 1;
#endif /* MDBX_ENABLE_PROFGC */
      TRACE("clear %s %s", "ALLOC_COALESCE", "since got threshold");
      if (pnl_size(txn->wr.repnl) >= num) {
        eASSERT0(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated &&
                          MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated);
        if (likely(num == 1)) {
          pgno = (flags & ALLOC_RESERVE) ? P_INVALID : repnl_get_single(txn);
          goto done;
        }
        pgno = gc_repnl_get_sequence(txn, num, flags);
        if (likely(pgno))
          goto done;
      }
    }
    flags &= ~(ALLOC_COALESCE | ALLOC_SHOULD_SCAN);
    if (unlikely(/* list is too long already */ pnl_size(txn->wr.repnl) >= env->options.rp_augment_limit) &&
        ((/* not a slot-request from gc-update */ num &&
          /* have enough unallocated space */ txn->geo.upper >= txn->geo.first_unallocated + num &&
          monotime_since_cached(monotime_begin, &now_cache) + txn->wr.gc.spent >= env->options.gc_time_limit) ||
         gc_len + pnl_size(txn->wr.repnl) >= PAGELIST_LIMIT)) {
      /* Stop reclaiming to avoid large/overflow the page list. This is a rare
       * case while search for a continuously multi-page region in a large database,
       * see https://libmdbx.dqdkfa.ru/dead-github/issues/123 */
      NOTICE("stop reclaiming %s: %zu (current) + %zu "
             "(chunk) >= %zu, rp_augment_limit %u",
             likely(gc_len + pnl_size(txn->wr.repnl) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached"
                                                                       : "to avoid PNL overflow",
             pnl_size(txn->wr.repnl), gc_len, gc_len + pnl_size(txn->wr.repnl), env->options.rp_augment_limit);
      goto depleted_gc;
    }
  }

  /* Append PNL from GC record to wr.repnl */
  ret.err = pnl_need(&txn->wr.repnl, gc_len);
  if (unlikely(ret.err != MDBX_SUCCESS))
    goto fail;

  if (LOG_ENABLED(MDBX_LOG_EXTRA)) {
    DEBUG_EXTRA("readed GC-pnl txn %" PRIaTXN " root %" PRIaPGNO " len %zu, PNL", id, txn->dbs[FREE_DBI].root, gc_len);
    for (size_t i = gc_len; i; i--)
      DEBUG_EXTRA_PRINT(" %" PRIaPGNO, gc_pnl[i]);
    DEBUG_EXTRA_PRINT(", first_unallocated %u\n", txn->geo.first_unallocated);
  }

  /* Merge in descending sorted order */
#if MDBX_ENABLE_PROFGC
  const uint64_t merge_begin = osal_monotime();
#endif /* MDBX_ENABLE_PROFGC */
  pnl_merge(txn->wr.repnl, gc_pnl);
#if MDBX_ENABLE_PROFGC
  prof->pnl_merge.calls += 1;
  prof->pnl_merge.volume += pnl_size(txn->wr.repnl);
  prof->pnl_merge.time += osal_monotime() - merge_begin;
#endif /* MDBX_ENABLE_PROFGC */
  flags |= ALLOC_SHOULD_SCAN;
  if (CHECKS2_ENABLED()) {
    if (unlikely(!pnl_check(txn->wr.repnl, txn->geo.first_unallocated))) {
      ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid txn retired-list");
      ret.err = MDBX_CORRUPTED;
      goto fail;
    }
  } else {
    eASSERT1(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated));
  }
  eASSERT1(env, txn_dpl_check(txn));

  eASSERT0(env, pnl_size(txn->wr.repnl) == 0 || MDBX_PNL_MOST(txn->wr.repnl) < txn->geo.first_unallocated);
  if (MDBX_ENABLE_REFUND && pnl_size(txn->wr.repnl) &&
      unlikely(MDBX_PNL_MOST(txn->wr.repnl) == txn->geo.first_unallocated - 1)) {
    /* Refund suitable pages into "unallocated" space */
    txn_refund(txn);
  }
  eASSERT1(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));

  eASSERT0(env, op == MDBX_PREV || op == MDBX_NEXT);
  rkl_t *rkl = &txn->wr.gc.reclaimed;
  const char *rkl_name = "reclaimed";
  if (mc->dbi_state != txn->dbi_state && (MDBX_DEBUG > 0 || gc_may_clean_reclaimed(txn))) {
    gc->next = txn->cursors[FREE_DBI];
    txn->cursors[FREE_DBI] = gc;
    ret.err = cursor_del(gc, 0);
    txn->cursors[FREE_DBI] = gc->next;
    if (likely(ret.err == MDBX_SUCCESS)) {
      if (unlikely(txn->dbs[FREE_DBI].items == 0)) {
        flags &= ~ALLOC_COALESCE;
        txn->flags |= txn_gc_drained;
        op = MDBX_FIRST; /* для предотвращения ошибок из-за относительного перемещения курсора */
      }
      rkl = &txn->wr.gc.ready4reuse;
      rkl_name = "ready4reuse";
    } else {
      VERBOSE("gc-early-clean: err %d, repnl %zu, gc-height %u (%u branch, %u leafs)", ret.err, pnl_size(txn->wr.repnl),
              gc->tree->height, gc->tree->branch_pages, gc->tree->leaf_pages);
      if (unlikely(txn->flags & MDBX_TXN_ERROR))
        goto fail;
    }
  }
  ret.err = rkl_push(rkl, id);
  TRACE("%" PRIaTXN " len %zu pushed to rkl-%s, err %d", id, gc_len, rkl_name, ret.err);
  if (unlikely(ret.err != MDBX_SUCCESS))
    goto fail;

  if (flags & ALLOC_COALESCE) {
    eASSERT0(env, op == MDBX_PREV || op == MDBX_NEXT);
    if (pnl_size(txn->wr.repnl) < env->maxgc_large1page / 2) {
      TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, pnl_size(txn->wr.repnl));
      goto next_gc;
    }
    flags -= ALLOC_COALESCE;
  }

scan:
  if ((flags & ALLOC_RESERVE) && num < 2) {
    /* Если был нужен только slot/id для gc_reclaim_slot() или gc_reserve4stockpile() */
    TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "reserve-done", id, pnl_size(txn->wr.repnl));
    ret.err = MDBX_SUCCESS;
    goto reserve_done;
  }

  eASSERT0(env, flags & ALLOC_SHOULD_SCAN);
  eASSERT0(env, num > 0);
  if (pnl_size(txn->wr.repnl) >= num) {
    eASSERT0(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated &&
                      MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated);
    if (likely(num == 1)) {
      eASSERT0(env, !(flags & ALLOC_RESERVE));
      pgno = repnl_get_single(txn);
      goto done;
    }
    pgno = gc_repnl_get_sequence(txn, num, flags);
    if (likely(pgno))
      goto done;
  }
  flags -= ALLOC_SHOULD_SCAN;
  if ((txn->flags & txn_gc_drained) == 0) {
    TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, pnl_size(txn->wr.repnl));
    goto next_gc;
  }

depleted_gc:
  TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, pnl_size(txn->wr.repnl));
  txn->flags |= txn_gc_drained;
  if (flags & ALLOC_SHOULD_SCAN)
    goto scan;

  //-------------------------------------------------------------------------

  /* There is no suitable pages in the GC and to be able to allocate
   * we should CHOICE one of:
   *  - make a new steady checkpoint if reclaiming was stopped by
   *    the last steady-sync, or wipe it in the MDBX_UTTERLY_NOSYNC mode;
   *  - kick lagging reader(s) if reclaiming was stopped by ones of it.
   *  - extend the database file. */

  /* Will use new pages from the map if nothing is suitable in the GC. */
  newnext = txn->geo.first_unallocated + num;

  /* Does reclaiming stopped at the last steady point? */
  const meta_ptr_t recent = meta_recent(env, &txn->wr.troika);
  const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->wr.troika);
  if ((flags & ALLOC_UNIMPORTANT) == 0 && recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady &&
      txn->env->gc.detent == prefer_steady.txnid) {
    DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN "-%s", recent.txnid, durable_caption(recent.ptr_c),
          prefer_steady.txnid, durable_caption(prefer_steady.ptr_c));
    const pgno_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed);
    const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed);
    uint64_t eoos_timestamp;
    /* wipe the last steady-point if one of:
     *  - UTTERLY_NOSYNC mode AND auto-sync threshold is NOT specified
     *  - UTTERLY_NOSYNC mode AND free space at steady-point is exhausted
     * otherwise, make a new steady-point if one of:
     *  - auto-sync threshold is specified and reached;
     *  - upper limit of database size is reached;
     *  - database is full (with the current file size)
     *       AND auto-sync threshold it NOT specified */
    if (F_ISSET(env->flags, MDBX_UTTERLY_NOSYNC) &&
        ((autosync_threshold | autosync_period) == 0 || newnext >= prefer_steady.ptr_c->geometry.now)) {
      /* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode
       * without any auto-sync threshold(s). */
#if MDBX_ENABLE_PROFGC
      env->lck->pgops.gc_prof.wipes += 1;
#endif /* MDBX_ENABLE_PROFGC */
      ret.err = meta_wipe_steady(env, txn->env->gc.detent);
      DEBUG("gc-wipe-steady, rc %d", ret.err);
      if (unlikely(ret.err != MDBX_SUCCESS))
        goto fail;
      eASSERT0(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->wr.troika).ptr_c);
      goto retry_gc_refresh_detent;
    }
    if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) ||
        (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) &&
         osal_monotime() - eoos_timestamp >= autosync_period) ||
        newnext >= txn->geo.upper ||
        ((num == 0 || newnext >= txn->geo.end_pgno) && (autosync_threshold | autosync_period) == 0)) {
      /* make steady checkpoint. */
#if MDBX_ENABLE_PROFGC
      env->lck->pgops.gc_prof.flushes += 1;
#endif /* MDBX_ENABLE_PROFGC */
      meta_t meta = *recent.ptr_c;
      ret.err = dxb_sync_locked(env, env->flags & (MDBX_WRITEMAP | MDBX_NOMETASYNC), &meta, &txn->wr.troika);
      DEBUG("gc-make-steady, rc %d", ret.err);
      eASSERT0(env, ret.err != MDBX_RESULT_TRUE);
      if (unlikely(ret.err != MDBX_SUCCESS))
        goto fail;
      if (prefer_steady.ptr_c != meta_prefer_steady(env, &txn->wr.troika).ptr_c)
        goto retry_gc_refresh_detent;
      eASSERT0(env, env->incore);
    }
  }

  if (unlikely(true == atomic_load32(&env->lck->rdt_refresh_flag, mo_AcquireRelease)) && txn_gc_detent(txn))
    goto retry_gc_have_detent;

  /* Avoid kick lagging reader(s) if is enough unallocated space
   * at the end of database file. */
  if (!(flags & ALLOC_RESERVE) && newnext <= txn->geo.end_pgno) {
    eASSERT0(env, pgno == 0);
    goto done;
  }

  if (txn_basis_snapshot(txn) > txn->env->gc.detent && mvcc_kick_laggards(txn, txn->env->gc.detent, nullptr))
    goto retry_gc_refresh_detent;

  //---------------------------------------------------------------------------

no_gc:
  eASSERT0(env, pgno == 0);
#ifndef MDBX_ENABLE_BACKLOG_DEPLETED
#define MDBX_ENABLE_BACKLOG_DEPLETED 0
#endif /* MDBX_ENABLE_BACKLOG_DEPLETED*/
  if (MDBX_ENABLE_BACKLOG_DEPLETED && unlikely(!(txn->flags & txn_gc_drained))) {
    ret.err = MDBX_BACKLOG_DEPLETED;
    goto fail;
  }
  if (flags & ALLOC_RESERVE) {
    ret.err = MDBX_NOTFOUND;
    goto fail;
  }

  /* Will use new pages from the map if nothing is suitable in the GC. */
  newnext = txn->geo.first_unallocated + num;
  if (newnext <= txn->geo.end_pgno)
    goto done;

  if (newnext > txn->geo.upper || !txn->geo.grow_pv) {
    NOTICE("gc-alloc: next %zu > upper %" PRIaPGNO, newnext, txn->geo.upper);
    ret.err = MDBX_MAP_FULL;
    goto fail;
  }

  eASSERT0(env, newnext > txn->geo.end_pgno);
  const size_t grow_step = pv2pages(txn->geo.grow_pv);
  size_t aligned = pgno_ceil2os_pgno(env, (pgno_t)(newnext + grow_step - newnext % grow_step));

  if (aligned > txn->geo.upper)
    aligned = txn->geo.upper;
  eASSERT0(env, aligned >= newnext);

  const txnid_t oldest_cached = env->lck->cached_oldest_txnid.weak;
  const meta_ptr_t syncpoint = meta_prefer_steady(env, &txn->wr.troika);
  const txnid_t oldest_rescan = shapshot_oldest_force_rescan(txn);
  NOTICE("try growth datafile to %zu pages (+%zu), pending-txnid %" PRIu64 ", gc-reclaiming detent %" PRIu64
         " (syncpoint %s.%" PRIu64 ", oldest-reader cached %" PRIu64 " rescan %" PRIu64 " %+zi)",
         aligned, aligned - txn->geo.end_pgno, txn->txnid, env->gc.detent, syncpoint.is_steady ? "steady" : "weak",
         syncpoint.txnid, oldest_cached, oldest_rescan, (intptr_t)(oldest_rescan - oldest_cached));
  ret.err = dxb_resize(env, txn->geo.first_unallocated, (pgno_t)aligned, txn->geo.upper, implicit_grow);
  if (ret.err != MDBX_SUCCESS) {
    ERROR("unable growth datafile to %zu pages (+%zu), errcode %d", aligned, aligned - txn->geo.end_pgno, ret.err);
    goto fail;
  }
  env->txn->geo.end_pgno = (pgno_t)aligned;
  eASSERT0(env, pgno == 0);

  //---------------------------------------------------------------------------

done:
  ret.err = MDBX_SUCCESS;
  if (likely((flags & ALLOC_RESERVE) == 0)) {
    if (pgno) {
      eASSERT0(env, pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS);
      eASSERT1(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
    } else {
      pgno = txn->geo.first_unallocated;
      txn->geo.first_unallocated += (pgno_t)num;
      eASSERT0(env, txn->geo.first_unallocated <= txn->geo.end_pgno);
      eASSERT0(env, pgno >= NUM_METAS && pgno + num <= txn->geo.first_unallocated);
    }

    ret = page_alloc_finalize(env, txn, mc, pgno, num);
    if (unlikely(ret.err != MDBX_SUCCESS)) {
    fail:
      eASSERT0(env, ret.err != MDBX_SUCCESS);
      if ((txn->flags & MDBX_TXN_ERROR) == 0) {
        if (MDBX_ENABLE_REFUND && pnl_size(txn->wr.repnl) &&
            unlikely(MDBX_PNL_MOST(txn->wr.repnl) == txn->geo.first_unallocated - 1)) {
          /* Refund suitable pages into "unallocated" space */
          txn_refund(txn);
        }
        eASSERT1(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
      }
      int level;
      if (flags & ALLOC_UNIMPORTANT)
        level = MDBX_LOG_DEBUG;
      else if (flags & ALLOC_RESERVE)
        level = MDBX_LOG_NOTICE;
      else {
        txn->flags |= MDBX_TXN_ERROR;
        level = MDBX_LOG_ERROR;
      }
      if (LOG_ENABLED(level)) {
        if (num)
          debug_log(level, __func__, __LINE__,
                    "unable %s %zu, alloc-flags 0x%x, err %d, txn-flags "
                    "0x%x, re-list-len %zu, loose-count %zu, gc: height %u, "
                    "branch %zu, leaf %zu, large %zu, entries %zu\n",
                    (flags & ALLOC_RESERVE) ? "reserve" : "alloc", num, flags, ret.err, txn->flags,
                    pnl_size(txn->wr.repnl), txn->wr.loose_count, txn->dbs[FREE_DBI].height,
                    (size_t)txn->dbs[FREE_DBI].branch_pages, (size_t)txn->dbs[FREE_DBI].leaf_pages,
                    (size_t)txn->dbs[FREE_DBI].large_pages, (size_t)txn->dbs[FREE_DBI].items);
        else
          debug_log(level, __func__, __LINE__,
                    "unable fetch-slot, alloc-flags 0x%x, err %d, txn-flags "
                    "0x%x, re-list-len %zu, loose-count %zu, gc: height %u, "
                    "branch %zu, leaf %zu, large %zu, entries %zu\n",
                    flags, ret.err, txn->flags, pnl_size(txn->wr.repnl), txn->wr.loose_count, txn->dbs[FREE_DBI].height,
                    (size_t)txn->dbs[FREE_DBI].branch_pages, (size_t)txn->dbs[FREE_DBI].leaf_pages,
                    (size_t)txn->dbs[FREE_DBI].large_pages, (size_t)txn->dbs[FREE_DBI].items);
      }
      ret.page = nullptr;
    }
    if (num > 1)
      txn->wr.gc.spent += monotime_since_cached(monotime_begin, &now_cache);
  } else {
  reserve_done:
    DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num, num ? "RESERVE" : "SLOT", ret.err);
    ret.page = nullptr;
  }

#if MDBX_ENABLE_PROFGC
  prof->rtime_monotonic += osal_monotime() - monotime_begin;
#endif /* MDBX_ENABLE_PROFGC */
  return ret;
}

__hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) {
  MDBX_txn *const txn = mc->txn;
  cASSERT0(txn, mc->txn->flags & MDBX_TXN_DIRTY);
  cASSERT0(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));

  /* If there are any loose pages, just use them */
  while (likely(txn->wr.loose_pages)) {
#if MDBX_ENABLE_REFUND
    if (unlikely(txn->wr.loose_refund_wl > txn->geo.first_unallocated)) {
      txn_refund(txn);
      if (!txn->wr.loose_pages)
        break;
    }
#endif /* MDBX_ENABLE_REFUND */

    page_t *lp = txn->wr.loose_pages;
    MDBX_ASAN_UNPOISON_MEMORY_REGION(lp, txn->env->ps);
    VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
    txn->wr.loose_pages = page_next(lp);
    txn->wr.loose_count--;
    DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc), lp->pgno);
    cASSERT0(txn, lp->pgno < txn->geo.first_unallocated);
    cASSERT0(txn, lp->pgno >= NUM_METAS);
    VALGRIND_MAKE_MEM_UNDEFINED(page2payload(lp), page_space(txn->env));
    lp->txnid = txn->front_txnid;
    pgr_t ret = {lp, MDBX_SUCCESS};
    return ret;
  }

  if (likely(pnl_size(txn->wr.repnl) > 0))
    return page_alloc_finalize(txn->env, txn, mc, repnl_get_single(txn), 1);

  return gc_alloc_ex(mc, 1, ALLOC_DEFAULT);
}

int gc_put_init(MDBX_txn *txn, gcu_t *ctx) {
  memset(ctx, 0, offsetof(gcu_t, sequel));
  /* Размер куска помещающийся на одну отдельную "overflow" страницу, но с небольшим запасом сводобного места. */
  ctx->goodchunk = txn->env->maxgc_large1page - (txn->env->maxgc_large1page >> 4);
  rkl_init(&ctx->sequel);
#if MDBX_ENABLE_BIGFOOT
  ctx->bigfoot = txn->txnid;
#endif /* MDBX_ENABLE_BIGFOOT */
  return cursor_init(&ctx->cursor, txn, FREE_DBI);
}

void gc_put_destroy(gcu_t *ctx) { rkl_destroy(&ctx->sequel); }

static size_t gc_chunk_pages(const MDBX_txn *txn, const size_t chunk) {
  return largechunk_npages(txn->env, gc_chunk_bytes(chunk));
}

static int gc_peekid(const MDBX_val *key, txnid_t *id) {
  if (likely(key->iov_len == sizeof(txnid_t))) {
    *id = unaligned_peek_u64(4, key->iov_base);
    return MDBX_SUCCESS;
  }
  ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC key-length");
  return MDBX_CORRUPTED;
}

#if MDBX_DEBUG_GCU
#pragma push_macro("LOG_ENABLED")
#undef LOG_ENABLED
#define LOG_ENABLED(LVL)                                                                                               \
  unlikely(MDBX_DEBUG_GCU > 2 || (ctx->loop > 1 && (MDBX_DEBUG_GCU > 1 || LVL < MDBX_LOG_EXTRA)) ||                    \
           LVL <= globals.loglevel)
#endif /* MDBX_DEBUG_GCU */

MDBX_NOTHROW_PURE_FUNCTION static bool is_lifo(const MDBX_txn *txn) {
  return (txn->env->flags & MDBX_LIFORECLAIM) != 0;
}

MDBX_NOTHROW_PURE_FUNCTION MDBX_MAYBE_UNUSED static inline const char *dbg_prefix(const gcu_t *ctx) {
  return is_lifo(ctx->cursor.txn) ? "    lifo" : "    fifo";
}

MDBX_MAYBE_UNUSED static void dbg_id(gcu_t *ctx, txnid_t id) {
#if MDBX_DEBUG_GCU
  if (ctx->dbg.prev) {
    if (ctx->dbg.prev != id - 1) {
      if (ctx->dbg.n)
        DEBUG_EXTRA_PRINT("-%" PRIaTXN, ctx->dbg.prev);
      if (id)
        DEBUG_EXTRA_PRINT(" %" PRIaTXN, id);
      ctx->dbg.n = 0;
    } else
      ctx->dbg.n += 1;
  } else {
    DEBUG_EXTRA_PRINT(" %" PRIaTXN, id);
    ctx->dbg.n = 0;
  }
  ctx->dbg.prev = id;
#else
  (void)ctx;
  (void)id;
#endif /* MDBX_DEBUG_GCU */
}

MDBX_MAYBE_UNUSED static void dbg_dump_ids(gcu_t *ctx) {
#if MDBX_DEBUG_GCU
  if (LOG_ENABLED(MDBX_LOG_EXTRA)) {
    DEBUG_EXTRA("%s", "GC:");
    if (ctx->cursor.tree->items) {
      cursor_couple_t couple;
      MDBX_val key;
      int err = cursor_init(&couple.outer, ctx->cursor.txn, FREE_DBI);
      if (err != MDBX_SUCCESS)
        ERROR("%s(), %d", "cursor_init", err);
      else
        err = outer_first(&couple.outer, &key, nullptr);

      txnid_t id;
      while (err == MDBX_SUCCESS) {
        err = gc_peekid(&key, &id);
        if (unlikely(err == MDBX_SUCCESS)) {
          dbg_id(ctx, id);
          if (id > couple.outer.txn->env->gc.detent)
            break;
          err = outer_next(&couple.outer, &key, nullptr, MDBX_NEXT);
        }
      }
      dbg_id(ctx, 0);
      DEBUG_EXTRA_PRINT("%s\n", (id > couple.outer.txn->env->gc.detent) ? "..." : "");
    } else
      DEBUG_EXTRA_PRINT("%s\n", " empty");

    DEBUG_EXTRA("%s", "ready4reuse:");
    if (rkl_empty(&txn->wr.gc.ready4reuse))
      DEBUG_EXTRA_PRINT("%s\n", " empty");
    else {
      rkl_iter_t i = rkl_iterator(&txn->wr.gc.ready4reuse, false);
      txnid_t id = rkl_turn(&i, false);
      while (id) {
        dbg_id(ctx, id);
        id = rkl_turn(&i, false);
      }
      dbg_id(ctx, 0);
      DEBUG_EXTRA_PRINT("%s\n", "");
    }

    DEBUG_EXTRA("%s", "comeback:");
    if (rkl_empty(&ctx->cursor.txn->wr.gc.comeback))
      DEBUG_EXTRA_PRINT("%s\n", " empty");
    else {
      rkl_iter_t i = rkl_iterator(&ctx->cursor.txn->wr.gc.comeback, false);
      txnid_t id = rkl_turn(&i, false);
      while (id) {
        dbg_id(ctx, id);
        id = rkl_turn(&i, false);
      }
      dbg_id(ctx, 0);
      DEBUG_EXTRA_PRINT("%s\n", "");
    }
  }
#else
  (void)ctx;
#endif /* MDBX_DEBUG_GCU */
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline size_t gc_reclaimed_maxspan_chunk(MDBX_txn *txn,
                                                                                             gcu_t *ctx) {
  (void)ctx;
  /* Функция вычисляет размер куска списка возвращаемых/неиспользованных в GC страниц, который можно разместить
   * в последовательности смежных страниц, которая возможно есть в самом этом списке.
   *
   * С одной стороны, такое размещение позволяет обойтись меньшим кол-вом слотов при возврате страниц,
   * уменьшить кол-во итераций при резервировании, а также не дробить саму последовательность.
   *
   * Однако, при последующей переработке такая тактика допустима только при не-отложенной очистке GC. Иначе
   * последовательность смежных страниц останется в GC до завершения переработавшей эту запись транзакции. А затем, в
   * ходе обновления GC при фиксации транзакции, будет раздроблена или снова возвращена в GC. Таким образом, без
   * не-отложенной очистки, такая тактика способствует миграции последовательностей страниц внутрь структуры самой GC
   * делая его более громоздкой.
   *
   * Поэтому пока используем только при нехватке слотов. */
  const size_t maxspan = pnl_maxspan(txn->wr.repnl);
  cASSERT0(txn, maxspan > 0);
  size_t start_lp = txn->env->maxgc_large1page /* начало набольшой страницы, с заголовком */;
  size_t tail_lp =
      ((maxspan - 1) << txn->env->ps2ln) / sizeof(txnid_t) /* продолжение большой страницы, без заголовка */;
  size_t pages4span = maxspan /* кол-во страниц для размещения*/;
  size_t chunk = start_lp + tail_lp - pages4span;
  TRACE("maxspan %zu, chunk %zu: %zu (start_lp) + %zu (tail_lp) - %zu (pages4span))", maxspan, chunk, start_lp, tail_lp,
        pages4span);
  return chunk;
}

static int gc_clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) {
  int err = MDBX_SUCCESS;
  if (ctx->retired_stored) {
    do {
      MDBX_val key, val;
#if MDBX_ENABLE_BIGFOOT
      key.iov_base = &ctx->bigfoot;
#else
      key.iov_base = &txn->txnid;
#endif /* MDBX_ENABLE_BIGFOOT */
      key.iov_len = sizeof(txnid_t);
      const csr_t csr = cursor_seek(&ctx->cursor, &key, &val, MDBX_SET);
      if (csr.err == MDBX_SUCCESS && csr.exact) {
        ctx->retired_stored = 0;
        err = cursor_del(&ctx->cursor, 0);
        TRACE("== clear-4linear @%" PRIaTXN ", stockpile %zu, err %d", *(txnid_t *)key.iov_base, gc_stockpile(txn),
              err);
      } else
        err = (csr.err == MDBX_NOTFOUND) ? MDBX_SUCCESS : csr.err;
    }
#if MDBX_ENABLE_BIGFOOT
    while (!err && --ctx->bigfoot >= txn->txnid);
#else
    while (0);
#endif /* MDBX_ENABLE_BIGFOOT */
  }
  return err;
}

static int gc_touch(gcu_t *ctx) {
  cASSERT0(ctx->cursor.txn, is_pointed(&ctx->cursor) || ctx->cursor.txn->dbs[FREE_DBI].leaf_pages == 0);
  MDBX_val key, val;
  key.iov_base = val.iov_base = nullptr;
  key.iov_len = sizeof(txnid_t);
  val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->wr.retired_pages);
  ctx->cursor.flags |= z_gcu_preparation;
  int err = cursor_touch(&ctx->cursor, &key, &val);
  ctx->cursor.flags -= z_gcu_preparation;
  return err;
}

static inline int gc_reclaim_slot(MDBX_txn *txn, gcu_t *ctx) {
  (void)txn;
  return gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE | ALLOC_UNIMPORTANT).err;
}

static inline int gc_reserve4retired(MDBX_txn *txn, gcu_t *ctx, size_t sequence_length) {
  (void)txn;
  return gc_alloc_ex(&ctx->cursor, sequence_length, ALLOC_RESERVE | ALLOC_UNIMPORTANT).err;
}

static inline int gc_reserve4stockpile(MDBX_txn *txn, gcu_t *ctx) {
  (void)txn;
  return gc_alloc_ex(&ctx->cursor, 1, ALLOC_RESERVE | ALLOC_UNIMPORTANT).err;
}

static int gc_prepare_stockpile(MDBX_txn *txn, gcu_t *ctx, const size_t for_retired) {
  for (;;) {
    cASSERT0(txn, is_pointed(&ctx->cursor) || txn->dbs[FREE_DBI].leaf_pages == 0);

    const size_t for_cow = txn->dbs[FREE_DBI].height;
    const size_t for_rebalance = for_cow + 1 + (txn->dbs[FREE_DBI].height + 1ul >= txn->dbs[FREE_DBI].branch_pages);
    const size_t for_tree_before_touch = for_cow + for_rebalance;
    const size_t for_tree_after_touch = for_rebalance;
    const size_t for_all_before_touch = for_retired + for_tree_before_touch;
    const size_t for_all_after_touch = for_retired + for_tree_after_touch;

    if (likely(for_retired < 2 && gc_stockpile(txn) > for_all_before_touch))
      return MDBX_SUCCESS;

    TRACE(">> retired-stored %zu, retired-left %zi, stockpile %zu, now-need %zu (4list %zu, "
          "4cow %zu, 4tree %zu)",
          ctx->retired_stored, pnl_size(txn->wr.retired_pages) - ctx->retired_stored, gc_stockpile(txn),
          for_all_before_touch, for_retired, for_cow, for_tree_before_touch);

    int err = gc_touch(ctx);
    TRACE("== after-touch, stockpile %zu, err %d", gc_stockpile(txn), err);

    if (!MDBX_ENABLE_BIGFOOT && unlikely(for_retired > 1) && pnl_size(txn->wr.retired_pages) != ctx->retired_stored &&
        err == MDBX_SUCCESS) {
      if (unlikely(ctx->retired_stored)) {
        err = gc_clean_stored_retired(txn, ctx);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
        if (!ctx->retired_stored)
          continue;
      }
      err = gc_reserve4retired(txn, ctx, for_retired);
      TRACE("== after-4linear, stockpile %zu, err %d", gc_stockpile(txn), err);
      cASSERT1(&ctx->cursor, gc_stockpile(txn) >= for_retired || err != MDBX_SUCCESS);
    }

    while (gc_stockpile(txn) < for_all_after_touch && err == MDBX_SUCCESS)
      err = gc_reserve4stockpile(txn, ctx);

    TRACE("<< stockpile %zu, err %d, gc: height %u, branch %zu, leaf %zu, large "
          "%zu, entries %zu",
          gc_stockpile(txn), err, txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages,
          (size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages,
          (size_t)txn->dbs[FREE_DBI].items);
    return (err != MDBX_NOTFOUND) ? err : MDBX_SUCCESS;
  }
}

static int gc_prepare_stockpile4update(MDBX_txn *txn, gcu_t *ctx) { return gc_prepare_stockpile(txn, ctx, 0); }

static int gc_prepare_stockpile4retired(MDBX_txn *txn, gcu_t *ctx) {
  const size_t retired_whole = pnl_size(txn->wr.retired_pages);
  const intptr_t retired_left = retired_whole - ctx->retired_stored;
  size_t for_retired = 0;
  if (retired_left > 0) {
    if (unlikely(!ctx->retired_stored)) {
      /* Make sure last page of GC is touched and on retired-list */
      int err = outer_last(&ctx->cursor, nullptr, nullptr);
      if (unlikely(err != MDBX_SUCCESS) && err != MDBX_NOTFOUND)
        return err;
      for_retired += 1;
    }
    if (MDBX_ENABLE_BIGFOOT) {
      const size_t per_branch_page = txn->env->maxgc_per_branch;
      for_retired += (retired_left + ctx->goodchunk - 1) / ctx->goodchunk;
      for (size_t entries = for_retired; entries > 1; for_retired += entries)
        entries = (entries + per_branch_page - 1) / per_branch_page;
    } else
      for_retired += largechunk_npages(txn->env, retired_whole);
  }

  return gc_prepare_stockpile(txn, ctx, for_retired);
}

int gc_merge_loose(MDBX_txn *txn) {
  cASSERT0(txn, txn->wr.loose_count > 0);
  /* Return loose page numbers to wr.repnl, though usually none are left at this point.
   * The pages themselves remain in dirtylist. */
  if (unlikely(!(txn->dbi_state[FREE_DBI] & DBI_DIRTY)) && txn->wr.loose_count < 3 + (unsigned)txn->dbs->height * 2) {
    /* Put loose page numbers in wr.retired_pages, since unreasonable to return ones to wr.repnl. */
    TRACE("merge %zu loose-pages into %s-pages", txn->wr.loose_count, "retired");
    int err = pnl_need(&txn->wr.retired_pages, txn->wr.loose_count);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
      pnl_append_prereserved(txn->wr.retired_pages, lp->pgno);
      MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
      VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
    }
  } else {
    /* Room for loose pages + temp PNL with same */
    TRACE("merge %zu loose-pages into %s-pages", txn->wr.loose_count, "reclaimed");
    int err = pnl_need(&txn->wr.repnl, 2 * txn->wr.loose_count + 2);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    pnl_t loose = txn->wr.repnl + pnl_alloclen(txn->wr.repnl) - txn->wr.loose_count - 1;
    size_t count = 0;
    for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
      cASSERT0(txn, lp->flags == P_LOOSE);
      loose[++count] = lp->pgno;
      MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
      VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
    }
    cASSERT0(txn, count == txn->wr.loose_count);
    pnl_setsize(loose, count);
    pnl_sort(loose, txn->geo.first_unallocated);
    pnl_merge(txn->wr.repnl, loose);
  }

  /* filter-out list of dirty-pages from loose-pages */
  dpl_t *const dl = txn->wr.dirtylist;
  if (dl) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
    cASSERT0(txn, dl->sorted <= dl->length);
    size_t w = 0, sorted_out = 0;
    for (size_t r = w; ++r <= dl->length;) {
      page_t *dp = dl->items[r].ptr;
      cASSERT0(txn, dp->flags == P_LOOSE || is_modifable(txn, dp));
      cASSERT0(txn, dpl_endpgno(dl, r) <= txn->geo.first_unallocated);
      if ((dp->flags & P_LOOSE) == 0) {
        if (++w != r)
          dl->items[w] = dl->items[r];
      } else {
        cASSERT0(txn, dp->flags == P_LOOSE);
        sorted_out += dl->sorted >= r;
        if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP))
          page_shadow_release(txn->env, dp, 1);
      }
    }
    TRACE("filtered-out loose-pages from %zu -> %zu dirty-pages", dl->length, w);
    cASSERT0(txn, txn->wr.loose_count == dl->length - w);
    dl->sorted -= sorted_out;
    cASSERT0(txn, dl->sorted <= w);
    dpl_setlen(dl, w);
    dl->pages_including_loose -= txn->wr.loose_count;
    txn->wr.dirtyroom += txn->wr.loose_count;
    cASSERT0(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
                      (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
  } else {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
  }
  txn->wr.loose_pages = nullptr;
  txn->wr.loose_count = 0;
#if MDBX_ENABLE_REFUND
  txn->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
  return MDBX_SUCCESS;
}

static int gc_store_retired(MDBX_txn *txn, gcu_t *ctx) {
  int err;
  MDBX_val key, data;

#if MDBX_ENABLE_BIGFOOT
  size_t retired_before;
  bool should_retry;
  do {
    if (ctx->bigfoot > txn->txnid) {
      err = gc_clean_stored_retired(txn, ctx);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      cASSERT0(txn, ctx->bigfoot <= txn->txnid);
    }

    err = gc_prepare_stockpile4retired(txn, ctx);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated);
    retired_before = pnl_size(txn->wr.retired_pages);
    should_retry = false;
    ctx->retired_stored = 0;
    ctx->bigfoot = txn->txnid;
    do {
      if (ctx->retired_stored) {
        err = gc_prepare_stockpile4retired(txn, ctx);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
      }
      key.iov_len = sizeof(txnid_t);
      key.iov_base = &ctx->bigfoot;
      const size_t left_before = retired_before - ctx->retired_stored;
      const size_t chunk_hi = ((left_before | 3) > ctx->goodchunk && ctx->bigfoot < (MAX_TXNID - UINT32_MAX))
                                  ? ctx->goodchunk
                                  : (left_before | 3);
      data.iov_len = gc_chunk_bytes(chunk_hi);
      /* Ложные предупреждения coverity возникают из-за поддержки вставки массивов значений в режиме MDBX_MULTIPLE. */
      /* coverity[OVERRUN] */
      err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE);
      if (unlikely(err != MDBX_SUCCESS))
        return err;

#if MDBX_DEBUG && (defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__))
      /* Для предотвращения предупреждения Valgrind из mdbx_dump_val() вызванное через макрос DVAL_DEBUG() на выходе из
       * cursor_seek(MDBX_SET_KEY), которая вызывается как выше в цикле очистки, так и ниже в цикле заполнения
       * зарезервированных элементов. */
      memset(data.iov_base, 0xBB, data.iov_len);
#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */

      const size_t retired_after = pnl_size(txn->wr.retired_pages);
      const size_t left_after = retired_after - ctx->retired_stored;
      const size_t chunk = (left_after < chunk_hi) ? left_after : chunk_hi;
      should_retry = retired_before != retired_after && chunk < retired_after;
      if (likely(!should_retry)) {
        const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) ? left_before - chunk : ctx->retired_stored;
        pgno_t *const begin = txn->wr.retired_pages + at;
        /* MDBX_PNL_ASCENDING == false && LIFO == false:
         *  - the larger pgno is at the beginning of retired list
         *    and should be placed with the larger txnid.
         * MDBX_PNL_ASCENDING == true && LIFO == true:
         *  - the larger pgno is at the ending of retired list
         *    and should be placed with the smaller txnid. */
        const pgno_t save = *begin;
        *begin = (pgno_t)chunk;
        memcpy(data.iov_base, begin, data.iov_len);
        *begin = save;
        TRACE("%s: put-retired/bigfoot @ %" PRIaTXN " (slice #%u) #%zu [%zu..%zu] of %zu", dbg_prefix(ctx),
              ctx->bigfoot, (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, retired_before);
      }
      ctx->retired_stored += chunk;
    } while (ctx->retired_stored < pnl_size(txn->wr.retired_pages) && (++ctx->bigfoot, true));
  } while (unlikely(should_retry));
#else
  /* Write to last page of GC */
  key.iov_len = sizeof(txnid_t);
  key.iov_base = &txn->txnid;
  do {
    gc_prepare_stockpile4retired(txn, ctx);
    data.iov_len = MDBX_PNL_SIZEOF(txn->wr.retired_pages);
    err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

#if MDBX_DEBUG && (defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__))
    /* Для предотвращения предупреждения Valgrind из mdbx_dump_val()
     * вызванное через макрос DVAL_DEBUG() на выходе
     * из cursor_seek(MDBX_SET_KEY), которая вызывается как выше в цикле
     * очистки, так и ниже в цикле заполнения зарезервированных элементов. */
    memset(data.iov_base, 0xBB, data.iov_len);
#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */

    /* Retry if wr.retired_pages[] grew during the Put() */
  } while (data.iov_len < MDBX_PNL_SIZEOF(txn->wr.retired_pages));

  ctx->retired_stored = pnl_size(txn->wr.retired_pages);
  pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated);
  cASSERT0(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->wr.retired_pages));
  memcpy(data.iov_base, txn->wr.retired_pages, data.iov_len);

  TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), ctx->retired_stored, txn->txnid);
#endif /* MDBX_ENABLE_BIGFOOT */
  if (MDBX_DEBUG_GCU < 2 && LOG_ENABLED(MDBX_LOG_EXTRA)) {
    size_t i = ctx->retired_stored;
    DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", txn->txnid, txn->dbs[FREE_DBI].root, i);
    for (; i; i--)
      DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->wr.retired_pages[i]);
    DEBUG_EXTRA_PRINT("%s\n", ".");
  }
  return MDBX_SUCCESS;
}

static int gc_remove_rkl(MDBX_txn *txn, gcu_t *ctx, rkl_t *rkl) {
  while (!rkl_empty(rkl)) {
    txnid_t id = rkl_edge(rkl, is_lifo(txn));
    if (ctx->gc_first == id)
      ctx->gc_first = 0;
    cASSERT0(txn, id <= txn->env->lck->cached_oldest_txnid.weak);
#ifndef _MSC_VER
    MDBX_val key = {.iov_base = &id, .iov_len = sizeof(id)};
#else
    /* avoid MSVC crash and/or ICE */
    MDBX_val key;
    key.iov_base = &id;
    key.iov_len = sizeof(id);
#endif
    int err = cursor_seek(&ctx->cursor, &key, nullptr, MDBX_SET).err;
    cASSERT0(txn, id == rkl_edge(rkl, is_lifo(txn)));
    if (err == MDBX_NOTFOUND) {
      err = rkl_push(&txn->wr.gc.ready4reuse, rkl_pop(rkl, is_lifo(txn)));
      WARNING("unexpected %s for gc-id %" PRIaTXN ", ignore and continue, push-err %d", "MDBX_NOTFOUND", id, err);
      if (unlikely(MDBX_IS_ERROR(err)))
        return err;
      continue;
    }
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    err = gc_prepare_stockpile4update(txn, ctx);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if (unlikely(id != rkl_edge(rkl, is_lifo(txn)))) {
      TRACE("id %" PRIaTXN " not at edge, continue", id);
      continue;
    }
    err = cursor_del(&ctx->cursor, 0);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    ENSURE_OBJ(txn->env, id == rkl_pop(rkl, is_lifo(txn)));
    cASSERT0(txn, id <= txn->env->lck->cached_oldest_txnid.weak);
    err = rkl_push(&txn->wr.gc.ready4reuse, id);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    TRACE("id %" PRIaTXN " cleared and moved to ready4reuse", id);
  }
  return MDBX_SUCCESS;
}

static inline int gc_clear_reclaimed(MDBX_txn *txn, gcu_t *ctx) {
  return gc_remove_rkl(txn, ctx, &txn->wr.gc.reclaimed);
}

static inline int gc_clear_returned(MDBX_txn *txn, gcu_t *ctx) {
  ctx->return_reserved_lo = 0;
  ctx->return_reserved_hi = 0;
  return gc_remove_rkl(txn, ctx, &txn->wr.gc.comeback);
}

static int gc_push_sequel(MDBX_txn *txn, gcu_t *ctx, txnid_t id) {
  cASSERT0(txn, id > 0 && id <= txn->env->gc.detent);
  tASSERT1(txn, !rkl_contain(&txn->wr.gc.comeback, id) && !rkl_contain(&txn->wr.gc.ready4reuse, id));
  TRACE("id %" PRIaTXN ", return-left %zi", id, ctx->return_left);
  int err = rkl_push(&ctx->sequel, id);
  if (unlikely(err != MDBX_SUCCESS)) {
    if (err == MDBX_RESULT_TRUE) {
      ERROR("%s/%d: %s", "MDBX_PROBLEM", MDBX_PROBLEM, "unexpected duplicate(s) during rkl-push");
      err = MDBX_PROBLEM;
    }
    return err;
  }
  ctx->return_left -= ctx->goodchunk;
  return (ctx->return_left <= 0) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
}

/* Строит гистограмму длин последовательностей соседствующих/примыкающих страниц */
static void gc_dense_hist(MDBX_txn *txn, gcu_t *ctx) {
  memset(&ctx->dense_histogram, 0, sizeof(ctx->dense_histogram));
  size_t seqlen = 0, seqmax = 1;
  for (size_t i = 2; i <= pnl_size(txn->wr.repnl); ++i) {
    seqlen += 1;
    if (seqlen == ARRAY_LENGTH(ctx->dense_histogram.array) ||
        !MDBX_PNL_CONTIGUOUS(txn->wr.repnl[i - 1], txn->wr.repnl[i], 1)) {
      ctx->dense_histogram.array[seqlen - 1] += 1;
      seqmax = (seqmax >= seqlen) ? seqmax : seqlen;
      seqlen = 0;
    }
  }
  ctx->dense_histogram.array[seqlen] += 1;
  ctx->dense_histogram.end = (unsigned)((seqmax > seqlen) ? seqmax : seqlen + 1);
}

/* Оптимальным решением является использование всех доступных слотов/идентификаторов, при максимальном использовании
 * последовательностей длиной ближе к целевой средней длине, необходимой для размещения всех возвращаемых страниц.
 *
 * Если последовательностей нужной или большей длины хватает, то достаточно просто выполнить соответствующую нарезку.
 * Иначе поиск решения можно рассматривать как необходимую замену в наборе коротких (в том числе нулевых)
 * последовательностей/кусков более длинными. Сложность в том, что нужно учитывать возможность разбиения/деления длинных
 * последовательностей на несколько более коротких.
 *
 * Поэтому алгоритмически поиск решения выглядит как попытка сначала нарезать N кусков по L, а в случае неуспеха
 * попробовать комбинации X=1..N-1 кусков по L+1 и Y=N-X кусков длины L и меньше (при достижении объёма/суммы),
 * затем комбинаций X=1..N-1 кусков по L+2, Y=0..N-X кусков длины L-1 и Z=N-(X+Y) кусков длины L и меньше:
 *  - a=0..(V/(L+1)) кусков L+1, плюс хвост N-a длиной до L включительно;
 *  - b=0..(V/(L+2)) кусков L+2, a=0..(V/(L+1)) кусков L+1, плюс хвост N-b-a длиной до L включительно;
 *  - c=0..(V/(L+3)) кусков L+3, b=0..(V/(L+2)) кусков L+2, a=0..(V/(L+1)) кусков L+1,
 *    плюс хвост N-c-b-a длиной до L включительно;
 *  - и т.д.
 *
 * 1. начинаем с максимальной длины из гистограммы и спускаемся до L
 *    для каждого уровня начинаем с 0 кусков и одного из условий:
 *     - исчерпание/отсутствие кусков нужной длины,
 *     - либо до достижения объёма (что должно привести к возврату решения);
 * 2. сначала спускаемся рекурсивно вглубь, затем идем двоичным поиском 0 --> hi на каждом уровне;
 * 3. поиск/выделение кусков и восстановление/откат:
 *     - каждое выделение может быть дробным, т.е. образовывать осколки меньшего размера, которые могут использовать
 *       последующие шаги;
 *     - на каждом уровне нужна своя актуальная гистограмма;
 *     - два варианта: локальная копия гистограммы, либо "дельта" для отката
 *     - с "дельтой" очень много возни, в том числе условных переходов,
 *       поэтому проще делать локальные копии и целиком их откатывать.
 *
 * Максимальная потребность памяти на стеке sizeof(pgno_t)*L*L, где L = максимальная длина последовательности
 * учитываемой в гистограмме. Для L=31 получается порядка 4 Кб на стеке, что представляется допустимым, а отслеживание
 * более длинных последовательностей не представляется рациональным.
 *
 * Сложность можно оценить как O(H*N*log(N)), либо как O(H*V*log(N)), где:
 *  - H = высота гистограммы,
 *  - N = количество имеющихся слотов/идентификаторов,
 *  - V = объем/количество не помещающихся номеров страниц. */

typedef struct sr_state {
  unsigned left_slots;
  pgno_t left_volume;
  gc_dense_histogram_t hist;
} sr_state_t;

/* Пытается отъесть n кусков длиной len, двигаясь по гистограмме от больших элементов к меньшим. */
static bool consume_stack(sr_state_t *const st, const size_t len, size_t n) {
  ASSERT(len > 1 && n > 0);
  while (st->hist.end >= len) {
    if (st->hist.array[st->hist.end - 1] < 1)
      st->hist.end -= 1;
    else {
      if (st->hist.end > len)
        st->hist.array[st->hist.end - len - 1] += 1;
      st->hist.array[st->hist.end - 1] -= 1;
      if (--n == 0)
        return true;
    }
  }
  return false;
}

typedef struct sr_context {
  /* расход страниц / ёмкость кусков */
  pgno_t first_page, other_pages;
  /* длина последовательностей смежных страниц, при нарезке на куски соответствующей длины, имеющихся
   * слотов/идентификаторов хватит для размещения возвращаемых страниц. Нарезать куски большего размера, есть смысл
   * только если недостаточно последовательностей такой длины (с учетом более длинных, в том числе кратно длиннее). */
  pgno_t factor;
  /* результирующее решение */
  gc_dense_histogram_t *solution;
} sr_context_t;

/* Пытается покрыть остаток объёма и слотов, кусками длиной не более factor,
 * двигаясь по гистограмме от больших элементов к меньшим. */
static bool consume_remaining(const sr_context_t *const ct, sr_state_t *const st, size_t len) {
  pgno_t *const solution = ct->solution->array;
  while (len > ct->factor)
    solution[--len] = 0;
  solution[len - 1] = 0;
  if (unlikely(0 >= (int)st->left_volume))
    goto done;

  size_t per_chunk = ct->first_page + ct->other_pages * (len - 1);
  while (st->hist.end > 0 && st->left_slots > 0) {
    if (st->hist.array[st->hist.end - 1]) {
      solution[len - 1] += 1;
      if (st->hist.end > len)
        st->hist.array[st->hist.end - len - 1] += 1;
      st->hist.array[st->hist.end - 1] -= 1;
      st->left_slots -= 1;
      st->left_volume -= per_chunk;
      if (0 >= (int)st->left_volume) {
      done:
        while (--len)
          solution[len - 1] = 0;
        return true;
      }
    } else {
      st->hist.end -= 1;
      if (len > st->hist.end) {
        ASSERT(len == st->hist.end + 1);
        len = st->hist.end;
        per_chunk -= ct->other_pages;
        solution[len - 1] = 0;
      }
    }
  }
  return false;
}

/* Поиск оптимального решения путем жадного бинарного деления и рекурсивного спуска по уже посчитанной гистограмме. */
static bool solve_recursive(const sr_context_t *const ct, sr_state_t *const st, size_t len) {
  ASSERT(st->left_slots >= 1);
  size_t per_chunk = ct->first_page + ct->other_pages * (len - 1);
  if (len > ct->factor && st->left_slots > 1 && st->left_volume > per_chunk) {
    unsigned lo = 0, hi = st->left_slots - 1, n = lo;
    do {
      sr_state_t local = *st;
      if (n) {
        if (!consume_stack(&local, len, n)) {
          hi = n - 1;
          n = (hi + lo) / 2;
          continue;
        }
        ASSERT(local.left_slots > n);
        local.left_slots -= n;
        local.left_volume = (local.left_volume > n * per_chunk) ? local.left_volume - n * per_chunk : 0;
      }
      if (!solve_recursive(ct, &local, len - 1)) {
        lo = n + 1;
      } else if (n > lo && n < hi) {
        hi = n;
      } else {
        ct->solution->array[len - 1] = n;
        *st = local;
        return true;
      }
      n = (hi + lo + 1) / 2;
    } while (hi >= lo);
    return false;
  }

  return consume_remaining(ct, st, len);
}

static int gc_dense_solve(MDBX_txn *txn, gcu_t *ctx, gc_dense_histogram_t *const solution) {
  sr_state_t st = {
      .left_slots = rkl_len(&txn->wr.gc.ready4reuse), .left_volume = ctx->return_left, .hist = ctx->dense_histogram};
  ASSERT(st.left_slots > 0 && st.left_volume > 0 && pnl_size(txn->wr.repnl) > 0);
  if (unlikely(!st.left_slots || !st.left_volume)) {
    ERROR("%s/%d: %s", "MDBX_PROBLEM", MDBX_PROBLEM, "recursive-solving preconditions violated");
    return MDBX_PROBLEM;
  }

  const sr_context_t ct = {.factor = gc_chunk_pages(txn, (st.left_volume + st.left_slots - 1) / st.left_slots),
                           .first_page = /* на первой странице */ txn->env->maxgc_large1page +
                                         /* сама страница также будет израсходована */ 1,
                           .other_pages = /* на второй и последующих страницах */ txn->env->ps / sizeof(pgno_t) +
                                          /* каждая страница также будет израсходована */ 1,
                           .solution = solution};

  memset(solution, 0, sizeof(*solution));
  if (solve_recursive(&ct, &st, st.hist.end)) {
    const pgno_t *end = ARRAY_END(solution->array);
    while (end > solution->array && end[-1] == 0)
      --end;
    solution->end = (unsigned)(end - solution->array);

    /* проверяем решение */
    size_t items = 0, volume = 0;
    for (size_t i = 0, chunk = ct.first_page; i < solution->end; ++i) {
      items += solution->array[i];
      volume += solution->array[i] * chunk;
      chunk += ct.other_pages;
    }

    if (unlikely(volume < (size_t)ctx->return_left || items > rkl_len(&txn->wr.gc.ready4reuse))) {
      ASSERT(!"recursive-solving failure");
      ERROR("%s/%d: %s", "MDBX_PROBLEM", MDBX_PROBLEM, "recursive-solving failure");
      return MDBX_PROBLEM;
    }
    return MDBX_RESULT_TRUE;
  }

  /* решение НЕ найдено */
  return MDBX_RESULT_FALSE;
}

// int gc_solve_test(MDBX_txn *txn, gcu_t *ctx) {
//   gc_dense_histogram_t r;
//   gc_dense_histogram_t *const solution = &r;
//
//   sr_state_t st = {.left_slots = 5,
//                    .left_volume = 8463,
//                    .hist = {.end = 31, .array = {6493, 705, 120, 14, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
//                                                  0,    0,   0,   0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4}}};
//   ASSERT(st.left_slots > 0 && st.left_volume > 0 && pnl_size(txn->wr.repnl) > 0);
//   if (unlikely(!st.left_slots || !st.left_volume)) {
//     ERROR("%s/%d: %s", "MDBX_PROBLEM", MDBX_PROBLEM, "recursive-solving preconditions violated");
//     return MDBX_PROBLEM;
//   }
//
//   const sr_context_t ct = {.factor = gc_chunk_pages(txn, (st.left_volume + st.left_slots - 1) / st.left_slots),
//                            .first_page = /* на первой странице */ txn->env->maxgc_large1page +
//                                          /* сама страница также будет израсходована */ 1,
//                            .other_pages = /* на второй и последующих страницах */ txn->env->ps / sizeof(pgno_t) +
//                                           /* каждая страница также будет израсходована */ 1,
//                            .solution = solution};
//
//   memset(solution, 0, sizeof(*solution));
//   if (solve_recursive(&ct, &st, st.hist.end)) {
//     const pgno_t *end = ARRAY_END(solution->array);
//     while (end > solution->array && end[-1] == 0)
//       --end;
//     solution->end = (unsigned)(end - solution->array);
//
//     /* проверяем решение */
//     size_t items = 0, volume = 0;
//     for (size_t i = 0, chunk = ct.first_page; i < solution->end; ++i) {
//       items += solution->array[i];
//       volume += solution->array[i] * chunk;
//       chunk += ct.other_pages;
//     }
//
//     if (unlikely(volume < (size_t)ctx->return_left || items > rkl_len(&txn->wr.gc.ready4reuse))) {
//       ASSERT(!"recursive-solving failure");
//       ERROR("%s/%d: %s", "MDBX_PROBLEM", MDBX_PROBLEM, "recursive-solving failure");
//       return MDBX_PROBLEM;
//     }
//     return MDBX_RESULT_TRUE;
//   }
//
//   /* решение НЕ найдено */
//   return MDBX_RESULT_FALSE;
// }

/* Ищем свободные/неиспользуемые id в GC, чтобы затем использовать эти идентификаторы для возврата неиспользованных
 * остатков номеров страниц ранее изъятых из GC.
 *
 * Нехватка идентификаторов достаточно редкая ситуация, так как возвращается страниц обычно не более чем было извлечено.
 * Однако, больше идентификаторов может потребоваться в следующих ситуациях:
 *
 *  - ранее с БД работала старая версия libmdbx без поддержки BigFoot и поэтому были переработано очень длинные записи,
 *    возврат остатков которых потребует нарезки на несколько кусков.
 *
 *  - ранее было зафиксировано несколько транзакций, которые помещали в GC retired-списки близкие к максимальному
 *    размеру помещающемуся на одну листовую страницу, после чего текущая транзакция переработала все эти записи,
 *    но по совокупности операций эти страницы оказались лишними и теперь при возврате потребуют больше слотов
 *    из-за обеспечения резерва свободного места при нарезке на множество кусков.
 *
 * Таким образом, потребность в поиске возникает редко и в большинстве случаев необходимо найти 1-2 свободных
 * слота/идентификатора. Если же требуется много слотов, то нет смысла экономить на поиске. */
static int gc_search_holes(MDBX_txn *txn, gcu_t *ctx) {
  cASSERT0(txn, ctx->return_left > 0 && txn->env->gc.detent);
  cASSERT0(txn, rkl_empty(&txn->wr.gc.reclaimed));
  if (!ctx->gc_first) {
    ctx->gc_first = txn->env->gc.detent;
    if (txn->dbs[FREE_DBI].items) {
      MDBX_val key;
      int err = outer_first(&ctx->cursor, &key, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      err = gc_peekid(&key, &ctx->gc_first);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }
  }

  /* В LIFO режиме требуется поиск в направлении от новых к старым записям с углублением в потенциально
   * рыхлую/неоднородную структуру, с последующим её заполнением возвращаемыми страницами. */

  /* В FIFO режиме, поиск внутри GC может быть полезным при нелинейной переработке (которая пока не реализована),
   * когда будет переработан один из следующих MVCC-снимков без переработки предыдущего. Необходимая для этого
   * независимость (отсутствие пересечения) снимков по набору retired-страниц может сложиться при последовательности
   * пишущих транзакций изменяющих данные в структурно одних и тех же страницах b-tree.
   *
   * Однако, в текущем понимании, это крайне редкая ситуация, при которой также весьма вероятно наличие свободного
   * интервала в начале GC, а поэтому вероятность выигрыша от дополнительного поиска вперед стремится к нулю. Кроме
   * этого, из-за мизерной вероятности ситуаций в которых такой поиск будет работать, его крайне сложно тестировать
   * -- требуется разработка отдельного теста, который может быть достаточно хрупким, так как любая доработка
   * основного кода может требовать изменений/подстройки сценария теста.
   *
   * Поэтому пока, до появления явной необходимости и/или пользы, решено отказаться от поиска свободных слотов вглубь GC
   * в направлении от старых записей к новым, в том числе в режиме FIFO. */

  dbg_dump_ids(ctx);
  const intptr_t tail_space =
      (intptr_t)(((ctx->gc_first > UINT16_MAX) ? (size_t)UINT16_MAX : (size_t)ctx->gc_first - 1) * ctx->goodchunk);
  const txnid_t reasonable_deep =
      txn->env->maxgc_per_branch +
      2 * (txn->env->gc.detent - txnid_min(rkl_lowest(&txn->wr.gc.ready4reuse), rkl_lowest(&txn->wr.gc.comeback)));
  const txnid_t scan_threshold = (txn->env->gc.detent > reasonable_deep) ? txn->env->gc.detent - reasonable_deep : 0;

  txnid_t scan_hi = txn->env->gc.detent, scan_lo = INVALID_TXNID;
  if (!is_lifo(txn) && ctx->gc_first < txn->env->gc.detent &&
      txn->env->gc.detent - ctx->gc_first < ctx->cursor.tree->items) {
    scan_hi = ctx->gc_first;
    scan_lo = 0;
  }

  rkl_iter_t iter_ready4reuse, iter_comeback;
  rkl_find(&txn->wr.gc.ready4reuse, scan_hi, &iter_ready4reuse);
  rkl_find(&txn->wr.gc.comeback, scan_hi, &iter_comeback);
  rkl_hole_t hole_ready4reuse = rkl_hole(&iter_ready4reuse, true);
  rkl_hole_t hole_comeback = rkl_hole(&iter_comeback, true);
  txnid_t begin, end;
  /* Ищем свободные id в GC в направлении от конца (новых записей) к началу (старым записям). */
  do {
    TRACE("hole-ready4reuse %" PRIaTXN "..%" PRIaTXN ", hole-comeback %" PRIaTXN "..%" PRIaTXN ", scan-range %" PRIaTXN
          "..%" PRIaTXN,
          hole_ready4reuse.begin, hole_ready4reuse.end, hole_comeback.begin, hole_comeback.end, scan_lo, scan_hi);
    MDBX_val key;
    int err;
    end = txnid_min(scan_hi, txnid_min(hole_ready4reuse.end, hole_comeback.end));
    if (hole_comeback.begin >= end) {
      hole_comeback = rkl_hole(&iter_comeback, true);
      TRACE("turn-comeback %" PRIaTXN "..%" PRIaTXN, hole_comeback.begin, hole_comeback.end);
    } else if (hole_ready4reuse.begin >= end) {
      hole_ready4reuse = rkl_hole(&iter_ready4reuse, true);
      TRACE("turn-ready4reuse %" PRIaTXN "..%" PRIaTXN, hole_ready4reuse.begin, hole_ready4reuse.end);
    } else if (scan_lo >= end) {
      TRACE("turn-scan from %" PRIaTXN "..%" PRIaTXN, scan_lo, scan_hi);
      scan_hi = scan_lo - 1;
      if (scan_lo - end > 4) {
        scan_lo = end - 1;
        key.iov_base = &scan_lo;
        key.iov_len = sizeof(scan_lo);
        const csr_t csr = cursor_seek(&ctx->cursor, &key, nullptr, MDBX_SET_RANGE);
        if (csr.err != MDBX_NOTFOUND) {
          if (unlikely(csr.err != MDBX_SUCCESS))
            return csr.err;
        }
        scan_hi = end - csr.exact;
      }
      goto scan;
    } else {
      begin = txnid_max(scan_lo, txnid_max(hole_ready4reuse.begin, hole_comeback.begin));
      cASSERT0(txn, begin <= scan_hi && begin > 0);
      while (--end >= begin) {
        err = gc_push_sequel(txn, ctx, end);
        cASSERT0(txn, (ctx->return_left > 0) == (err != MDBX_RESULT_TRUE));
        if (err != MDBX_SUCCESS) {
          return err;
        }
      }
      if (MIN_TXNID >= begin)
        break;
      if (begin == hole_comeback.begin) {
        hole_comeback = rkl_hole(&iter_comeback, true);
        TRACE("pull-comeback %" PRIaTXN "..%" PRIaTXN, hole_comeback.begin, hole_comeback.end);
      }
      if (begin == hole_ready4reuse.begin) {
        hole_ready4reuse = rkl_hole(&iter_ready4reuse, true);
        TRACE("pull-ready4reuse %" PRIaTXN "..%" PRIaTXN, hole_ready4reuse.begin, hole_ready4reuse.end);
      }
      if (begin == scan_lo) {
        TRACE("pull-scan from %" PRIaTXN "..%" PRIaTXN, scan_lo, scan_hi);
        do {
          scan_hi = scan_lo - 1;
        scan:
          if (scan_hi < scan_threshold && tail_space >= ctx->return_left) {
            /* Искать глубже нет смысла, ибо в начале GC есть достаточно свободных идентификаторов. */
            TRACE("stop-scan %s", "threshold");
            scan_lo = 0;
            scan_hi = ctx->gc_first;
            break;
          }
          err = outer_prev(&ctx->cursor, &key, nullptr, MDBX_PREV);
          if (err == MDBX_NOTFOUND) {
            /* больше нет записей ближе к началу GC, все значения id свободны */
            TRACE("stop-scan %s", "eof");
            scan_lo = 0;
            break;
          }
          if (unlikely(err != MDBX_SUCCESS))
            return err;
          err = gc_peekid(&key, &scan_lo);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
          TRACE("scan: peek %" PRIaTXN, scan_lo);
          scan_lo += 1;
        } while (scan_lo >= scan_hi);
        TRACE("scan-range %" PRIaTXN "..%" PRIaTXN, scan_lo, scan_hi);
      }
    }
  } while (end > MIN_TXNID);
  return MDBX_SUCCESS;
}

static inline int gc_reserve4return(MDBX_txn *txn, gcu_t *ctx, const size_t chunk_lo, const size_t chunk_hi) {
  txnid_t reservation_id = rkl_pop(&txn->wr.gc.ready4reuse, true);
  TRACE("%s: slots-ready4reuse-left %zu, reservation-id %" PRIaTXN, dbg_prefix(ctx), rkl_len(&txn->wr.gc.ready4reuse),
        reservation_id);
  cASSERT0(txn, reservation_id >= MIN_TXNID && reservation_id < txn->txnid);
  cASSERT0(txn, reservation_id <= txn->env->lck->cached_oldest_txnid.weak);
  if (unlikely(reservation_id < MIN_TXNID ||
               reservation_id > atomic_load64(&txn->env->lck->cached_oldest_txnid, mo_Relaxed))) {
    ERROR("** internal error (reservation gc-id %" PRIaTXN ")", reservation_id);
    return MDBX_PROBLEM;
  }

  int err = rkl_push(&txn->wr.gc.comeback, reservation_id);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

#ifndef _MSC_VER
  MDBX_val key = {.iov_base = &reservation_id, .iov_len = sizeof(reservation_id)};
#else
  /* avoid MSVC crash and/or ICE */
  MDBX_val key;
  key.iov_base = &reservation_id;
  key.iov_len = sizeof(reservation_id);
#endif
  MDBX_val data = {.iov_base = nullptr, .iov_len = gc_chunk_bytes(chunk_hi)};
  TRACE("%s: reserved +%zu...+%zu [%zu...%zu), err %d", dbg_prefix(ctx), chunk_lo, chunk_hi,
        ctx->return_reserved_lo + 1, ctx->return_reserved_hi + chunk_hi + 1, err);
  gc_prepare_stockpile4update(txn, ctx);
  /* LY: ложные предупреждения coverity возникают из-за поддержки вставки массивов значений в режиме MDBX_MULTIPLE */
  /* coverity[OVERRUN] */
  err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE);
  tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  memset(data.iov_base, 0, data.iov_len);
  ctx->return_reserved_lo += chunk_lo;
  ctx->return_reserved_hi += chunk_hi;
  if (unlikely(!rkl_empty(&txn->wr.gc.reclaimed))) {
    NOTICE("%s: restart since %zu slot(s) reclaimed (reserved %zu...%zu of %zu)", dbg_prefix(ctx),
           rkl_len(&txn->wr.gc.reclaimed), ctx->return_reserved_lo, ctx->return_reserved_hi, pnl_size(txn->wr.repnl));
    return MDBX_RESULT_TRUE;
  }

  return MDBX_SUCCESS;
}

static size_t dense_chunk_outlay(const MDBX_txn *txn, const size_t chunk) {
  size_t need_span = gc_chunk_pages(txn, chunk);
  return gc_repnl_has_span(txn, need_span) ? need_span : 0;
}

static size_t dense_adjust_chunk(const MDBX_txn *txn, const size_t chunk) {
  size_t adjusted = chunk;
  if (chunk > txn->env->maxgc_large1page) {
    size_t hi = chunk + 1, lo = chunk - gc_chunk_pages(txn, chunk) - 1;
    while (lo < hi) {
      adjusted = (hi + lo) / 2;
      size_t probe = chunk - dense_chunk_outlay(txn, adjusted);
      if (probe > adjusted)
        lo = adjusted + 1;
      else if (probe < adjusted)
        hi = adjusted - 1;
      else
        break;
    }
  }
  return adjusted;
}

static size_t dense_adjust_amount(const MDBX_txn *const txn, size_t amount) {
  const size_t gap = 2 + txn->dbs[FREE_DBI].height;
  const size_t snubber = txn->env->ps / sizeof(pgno_t) / 2;
  return ((amount + gap < txn->env->maxgc_large1page) ? txn->env->maxgc_large1page : amount + snubber);
}

static int gc_handle_dense(MDBX_txn *txn, gcu_t *ctx, size_t left_min, size_t left_max) {
  /* Крайне маловероятная ситуация, в текущем понимании не возможная при нормальной/ожидаемой работе всех
   * актуальных версий движка. Тем не менее, сюда мы можем попасть при использовании БД с содержимым GC
   * оставшимся после старых версий и/или при выключенном BigFoot. Тогда в GC могут быть записи огромного
   * размера, при возврате которых мы получаем так много кусков, что в GC не хватает свободных/неиспользуемых
   * идентификаторов от прошлых транзакций.
   *
   * Дальше три возможности:
   * 1. Искать в GC доступные для переработки записи короче maxgc_large1page. Это малоэффективный путь,
   *    так как если мы уже попали в текущую ситуацию, то маловероятно что в GC есть такие записи и что запаса
   *    места хватит. Поэтому оставляем этот путь в качестве предпоследнего варианта.
   * 2. Попытаться запихнуть остаток одним куском, который может быть многократно больше maxgc_large1page,
   *    т.е. потребуется несколько последовательных свободных страниц, из-за чего может произойти загрузка всей
   *    GC и т.д. Это плохой путь, который можно использовать только в качестве последнего шанса.
   * 3. Искать смежные страницы среди возвращаемых и сохранять куски помещающиеся в такие последовательности.
   *
   *  Поэтому комбинируем все три варианта 3+1+2:
   *  - Вычисляем среднюю целевую длину куска в large/overflow страницах, при нарезке на которые имеющихся
   *    слотов/идентификаторов хватит для размещения возвращаемых страниц.
   *  - В идеале ищем в wr.repnl последовательности смежных страниц длиной от ⌊целевой длины куска⌋
   *    до ⌈целевой длины куска⌉ и выполняем резервирование кусками помещающимися в эти последовательности.
   *    Теоретически, вероятность (а следовательно и количество) последовательностей экспоненциально уменьшается
   *    с увеличением длины. На практике, в основном, это будут пары и тройки страниц, но также и длинные
   *    последовательности, которые образуются в исходных больших транзакциях (порождающий большие
   *    retired-списки), особенно при выделении новых страниц. При этом использование длинных
   *    последовательностей чревато повторением проблем при переработке созданных сейчас записей.
   *  - Поэтому оптимальное решение выглядит как поиск набора последовательностей, мощность которого равна
   *    количеству доступных слотов/идентификаторов, а длины последовательностей минимальны, но достаточны для
   *    размещения всех возвращаемых страниц. */

  int err = MDBX_RESULT_FALSE;
  if (!rkl_empty(&txn->wr.gc.ready4reuse)) {
    gc_dense_hist(txn, ctx);
    gc_dense_histogram_t solution;
    if (ctx->loop == 1 || ctx->loop % 3 == 0)
      left_max = dense_adjust_amount(txn, left_max);
    ctx->return_left = left_max;
    err = gc_dense_solve(txn, ctx, &solution);
    if (err == MDBX_RESULT_FALSE /* решение НЕ найдено */ && left_max != left_min) {
      if (ctx->loop == 1 || ctx->loop % 3 == 0)
        left_min = dense_adjust_amount(txn, left_min);
      if (left_max != left_min) {
        ctx->return_left = left_min;
        err = gc_dense_solve(txn, ctx, &solution);
      }
    }
    if (err == MDBX_RESULT_TRUE /* решение найдено */) {
      for (size_t i = solution.end; i > 0; --i)
        for (pgno_t n = 0; n < solution.array[i - 1]; ++n) {
          size_t span = i;
          size_t chunk_hi = txn->env->maxgc_large1page + txn->env->ps / sizeof(pgno_t) * (span - 1);
          if (chunk_hi > left_max) {
            chunk_hi = left_max;
            span = gc_chunk_pages(txn, chunk_hi);
          }
          size_t chunk_lo = chunk_hi - txn->env->maxgc_large1page + ctx->goodchunk;
          TRACE("%s: dense-chunk (seq-len %zu, %d of %d) %zu...%zu, gc-per-ovpage %u", dbg_prefix(ctx), i, n + 1,
                solution.array[i - 1], chunk_lo, chunk_hi, txn->env->maxgc_large1page);
          size_t amount = pnl_size(txn->wr.repnl);
          err = gc_reserve4return(txn, ctx, chunk_lo, chunk_hi);
          if (unlikely(err != MDBX_SUCCESS))
            return err;

          const size_t now = pnl_size(txn->wr.repnl);
          if (span < amount - now - txn->dbs[FREE_DBI].height || span > amount - now + txn->dbs[FREE_DBI].height)
            TRACE("dense-%s-reservation: miss %zu (expected) != %zi (got)", "solve", span, amount - now);
          amount = now;
          if (ctx->return_reserved_hi >= amount)
            return MDBX_SUCCESS;
          left_max = dense_adjust_amount(txn, amount) - ctx->return_reserved_lo;
        }
    }
  } else if (rkl_len(&txn->wr.gc.comeback)) {
    NOTICE("%s: restart since %zu slot(s) comeback non-dense (reserved %zu...%zu of %zu)", dbg_prefix(ctx),
           rkl_len(&txn->wr.gc.comeback), ctx->return_reserved_lo, ctx->return_reserved_hi, pnl_size(txn->wr.repnl));
    return /* повтор цикла */ MDBX_RESULT_TRUE;
  }

  if (err == MDBX_RESULT_FALSE /* решение НЕ найдено, либо нет идентификаторов */) {
    if (ctx->return_left > (intptr_t)txn->env->maxgc_large1page) {
      err = gc_reclaim_slot(txn, ctx);
      if (err == MDBX_NOTFOUND)
        err = gc_reserve4retired(txn, ctx, gc_chunk_pages(txn, dense_adjust_chunk(txn, ctx->return_left)));
      if (err != MDBX_NOTFOUND && err != MDBX_SUCCESS)
        return err;
    }

    const size_t per_page = txn->env->ps / sizeof(pgno_t);
    size_t amount = pnl_size(txn->wr.repnl);
    do {
      if (rkl_empty(&txn->wr.gc.ready4reuse)) {
        NOTICE("%s: restart since no slot(s) available (reserved %zu...%zu of %zu)", dbg_prefix(ctx),
               ctx->return_reserved_lo, ctx->return_reserved_hi, amount);
        return MDBX_RESULT_TRUE;
      }
      const size_t left = dense_adjust_amount(txn, amount) - ctx->return_reserved_hi;
      const size_t slots = rkl_len(&txn->wr.gc.ready4reuse);
      const size_t base = (left + slots - 1) / slots;
      const size_t adjusted = dense_adjust_chunk(txn, base);
      TRACE("dense-reservation: reserved %zu...%zu of %zu, left %zu slot(s) and %zu pnl, step: %zu base,"
            " %zu adjusted",
            ctx->return_reserved_lo, ctx->return_reserved_hi, amount, slots, left, base, adjusted);
      const size_t chunk_hi =
          (adjusted > txn->env->maxgc_large1page)
              ? txn->env->maxgc_large1page + ceil_powerof2(adjusted - txn->env->maxgc_large1page, per_page)
              : txn->env->maxgc_large1page;
      const size_t chunk_lo =
          (adjusted > txn->env->maxgc_large1page)
              ? txn->env->maxgc_large1page + floor_powerof2(adjusted - txn->env->maxgc_large1page, per_page)
              : adjusted;
      err = gc_reserve4return(txn, ctx, chunk_lo, chunk_hi);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      const size_t now = pnl_size(txn->wr.repnl);
      if (base - adjusted + txn->dbs[FREE_DBI].height < amount - now ||
          base - adjusted > amount - now + txn->dbs[FREE_DBI].height)
        TRACE("dense-%s-reservation: miss %zu (expected) != %zi (got)", "unsolve", base - adjusted, amount - now);
      amount = now;
    } while (ctx->return_reserved_hi < amount);
  }

  if (unlikely(err != MDBX_SUCCESS))
    ERROR("unable provide IDs and/or to fit returned PNL (%zd+%zd pages, %zd+%zd slots), err %d", ctx->retired_stored,
          pnl_size(txn->wr.repnl), rkl_len(&txn->wr.gc.comeback), rkl_len(&txn->wr.gc.ready4reuse), err);
  return err;
}

/* Выполняет один шаг резервирования записей для возврата в GC страниц (их номеров), оставшихся после
 * переработки GC и последующего использования в транзакции. */
static int gc_rerere(MDBX_txn *txn, gcu_t *ctx) {
  /* При резервировании часть оставшихся страниц может быть использована до полного исчерпания остатка,
   * что также может привести к переработке дополнительных записей GC. Таким образом, на каждой итерации
   * ситуация может существенно меняться, в том числе может потребоваться очистка резерва и повтор всего цикла.
   *
   * Кроме этого, теоретически в GC могут быть очень большие записи (созданные старыми версиями движка и/или
   * при выключенной опции MDBX_ENABLE_BIGFOOT), которые при возврате будут нарезаться на более мелкие куски.
   * В этом случае возвращаемых записей будет больше чем было переработано, поэтому потребуются дополнительные
   * идентификаторы/слоты отсутствующие в GC. */

  // gc_solve_test(txn, ctx);

  cASSERT0(txn, rkl_empty(&txn->wr.gc.reclaimed));
  const size_t amount = pnl_size(txn->wr.repnl);
  if (ctx->return_reserved_hi >= amount) {
    if (unlikely(ctx->dense)) {
      ctx->dense = false;
      NOTICE("%s: out of dense-mode (amount %zu, reserved %zu..%zu)", dbg_prefix(ctx), amount, ctx->return_reserved_lo,
             ctx->return_reserved_hi);
    }
    if (unlikely(amount ? (amount + txn->env->maxgc_large1page < ctx->return_reserved_lo)
                        : (ctx->return_reserved_hi > 3))) {
      /* после резервирования было израсходованно слишком много страниц и получилось слишком много резерва */
      TRACE("%s: reclaimed-list %zu < reversed %zu..%zu, retry", dbg_prefix(ctx), amount, ctx->return_reserved_lo,
            ctx->return_reserved_hi);
      return MDBX_RESULT_TRUE;
    }
    /* резерва достаточно, ничего делать не надо */
    return MDBX_SUCCESS;
  }

  const size_t left_min = amount - ctx->return_reserved_hi;
  const size_t left_max = amount - ctx->return_reserved_lo;
  if (likely(left_min < txn->env->maxgc_large1page && !rkl_empty(&txn->wr.gc.ready4reuse))) {
    /* Есть хотя-бы один слот и весь остаток списка номеров страниц помещается в один кусок.
     * Это самая частая ситуация, просто продолжаем. */
  } else {
    if (likely(rkl_len(&txn->wr.gc.ready4reuse) * ctx->goodchunk >= left_max)) {
      /* Слотов хватает, основная задача делить на куски так, чтобы изменение (уменьшение) кол-ва возвращаемых страниц в
       * процессе резервирования записей в GC не потребовало менять резервирование, т.е. удалять и повторять всё снова.
       */
    } else {
      /* Слотов нет, либо не хватает для нарезки возвращаемых страниц кусками по goodchunk */
      ctx->return_left = left_max;
      int err = gc_search_holes(txn, ctx);
      cASSERT0(txn, (ctx->return_left <= 0) == (err == MDBX_RESULT_TRUE));
      if (unlikely(MDBX_IS_ERROR(err)))
        return err;

      if (!rkl_empty(&ctx->sequel)) {
        err = rkl_merge(&ctx->sequel, &txn->wr.gc.ready4reuse, false);
        if (unlikely(err != MDBX_SUCCESS)) {
          if (err == MDBX_RESULT_TRUE) {
            ERROR("%s/%d: %s", "MDBX_PROBLEM", MDBX_PROBLEM, "unexpected duplicate(s) during rkl-merge");
            err = MDBX_PROBLEM;
          }
          return err;
        }
        rkl_clear(&ctx->sequel);
      }

      if (unlikely(ctx->return_left > 0)) {
        /* Делаем переоценку баланса для кусков предельного размера (по maxgc_large1page, вместо goodchunk). */
        const intptr_t dense_unfit = left_min - rkl_len(&txn->wr.gc.ready4reuse) * txn->env->maxgc_large1page;
        if (dense_unfit > 0) {
          /* Имеющихся идентификаторов НЕ хватит,
           * даже если если их использовать для кусков размером maxgc_large1page вместо goodchunk. */
          if (!ctx->dense) {
            NOTICE("%s: enter to dense-mode (amount %zu, reserved %zu..%zu, slots/ids %zu, left %zu..%zu, unfit %zu)",
                   dbg_prefix(ctx), amount, ctx->return_reserved_lo, ctx->return_reserved_hi,
                   rkl_len(&txn->wr.gc.ready4reuse), left_min, left_max, dense_unfit);
            ctx->dense = true;
          }
          return gc_handle_dense(txn, ctx, left_min, left_max);
        }
      }
      cASSERT0(txn, rkl_empty(&txn->wr.gc.reclaimed));
    }
  }

  /* Максимальный размер куска, который помещается на листовой странице, без выноса на отдельную "overflow" страницу. */
  const size_t chunk_inpage = (txn->env->leaf_nodemax - NODESIZE - sizeof(txnid_t)) / sizeof(pgno_t) - 1;

  /* Размер куска помещающийся на одну отдельную "overflow" страницу, но с небольшим запасом сводобного места. */
  const size_t chunk_good = ctx->goodchunk;

  /* Учитываем резервирование по минимальному размеру кусков (chunk_lo), но резервируем слоты с некоторым запасом
   * (chunk_hi). При этом предполагая что каждый слот может быть заполнен от chunk_lo до chunk_hi, что обеспечивает
   * хорошую амортизацию изменения размера списка возвращаемых страниц, как из-за расходов на создаваемые записи, так и
   * из-за переработки GC. */
  const size_t chunk_lo = (left_min < chunk_inpage) ? left_min : chunk_good;
  /* Куски размером больше chunk_inpage и до maxgc_large1page включительно требуют одной "overflow" страницы.
   * Соответственно требуют одинаковых затрат на обслуживание, а диапазон между chunk_good и maxgc_large1page
   * амортизирует изменения кол-ва списка возвращаемых страниц.
   *
   * Выравниваем размер коротких кусков на 4 (т.е. до 3, с учетом нулевого элемента с длиной),
   * а длинных кусков до maxgc_large1page */
  const size_t chunk_hi = (((left_max + 1) | 3) > chunk_inpage) ? txn->env->maxgc_large1page : ((left_max + 1) | 3);

  TRACE("%s: chunk %zu...%zu, gc-per-ovpage %u", dbg_prefix(ctx), chunk_lo, chunk_hi, txn->env->maxgc_large1page);
  cASSERT0(txn, chunk_lo > 0 && chunk_lo <= chunk_hi && chunk_hi > 1);
  return gc_reserve4return(txn, ctx, chunk_lo, chunk_hi);
}

/* Заполняет зарезервированные записи номерами возвращаемых в GC страниц. */
static int gc_fill_returned(MDBX_txn *txn, gcu_t *ctx) {
  tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
  tASSERT1(txn, txn_dpl_check(txn));

  /* Уже есть набор зарезервированных записей GC, id которых собраны в txn->wr.gc.comeback. При этом текущее
   * кол-вол возвращаемых страниц (оставшихся после расходов на резервирование) точно помещается в
   * эти записи и скорее всего с некоторым запасом. Иначе, если резерва недостаточно или избыток
   * резерва неприемлемо велик, то нет другого способа как удалить все созданные записи и повторить
   * всё ещё раз, и дальше этот путь здесь не рассматривается.
   *
   * В большинстве случаев, при резервировании записей переработка GC происходить не будет. Поэтому
   * размер резервированных записей кроме последней будет равен gc_largechunk_preferred_size(),
   * а последней округлённому/выровненному остатку страниц. Однако, в общем случае, может существенно
   * колебаться как размер записей, так и "баланс" отклонения от среднего.
   *
   * Если считать что резерва достаточно и имеющийся избыток допустим, то задача заполнения сводится
   * к распределению излишков резерва по записям с учётом их размера, а далее просто к записи данных.
   * При этом желательно обойтись без каких-то сложных операций типа деления и т.п. */
  const size_t amount = pnl_size(txn->wr.repnl);
  cASSERT0(txn, amount > 0 && amount <= ctx->return_reserved_hi && !rkl_empty(&txn->wr.gc.comeback));
  const size_t slots = rkl_len(&txn->wr.gc.comeback);
  if (likely(slots == 1)) {
    /* самый простой и частый случай */
    txnid_t id = rkl_lowest(&txn->wr.gc.comeback);
#ifndef _MSC_VER
    MDBX_val key = {.iov_base = &id, .iov_len = sizeof(id)};
#else
    /* avoid MSVC crash and/or ICE */
    MDBX_val key;
    key.iov_base = &id;
    key.iov_len = sizeof(id);
#endif
    MDBX_val data = {.iov_base = nullptr, .iov_len = 0};
    int err = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err;
    if (likely(err == MDBX_SUCCESS)) {
      pgno_t *const from = MDBX_PNL_BEGIN(txn->wr.repnl), *const to = MDBX_PNL_END(txn->wr.repnl);
      TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN " (%s)", dbg_prefix(ctx),
            pnl_size(txn->wr.repnl), from - txn->wr.repnl, from[0], to - txn->wr.repnl, to[-1], id, "at-once");
      cASSERT0(txn, data.iov_len >= gc_chunk_bytes(pnl_size(txn->wr.repnl)));
      if (unlikely(data.iov_len - gc_chunk_bytes(pnl_size(txn->wr.repnl)) >= txn->env->ps * 2)) {
        NOTICE("too long %s-comeback-reserve @%" PRIaTXN ", have %zu bytes, need %zu bytes", "single", id, data.iov_len,
               gc_chunk_bytes(pnl_size(txn->wr.repnl)));
        return MDBX_RESULT_TRUE;
      }
      /* coverity[var_deref_model] */
      memcpy(data.iov_base, txn->wr.repnl, gc_chunk_bytes(pnl_size(txn->wr.repnl)));
    }
    return err;
  }

  rkl_iter_t iter = rkl_iterator(&txn->wr.gc.comeback, is_lifo(txn));
  size_t surplus = ctx->return_reserved_hi - amount, stored = 0;
  const uint64_t factor = ((uint64_t)surplus << 32) / ctx->return_reserved_hi;
  ratio2digits_buffer_t factor_rough, factor_sharp;
  TRACE("%s: amount %zu, slots %zu, surplus %zu (%zu..%zu), factor %s (%" PRIu64 " >> 32, sharp %s)", dbg_prefix(ctx),
        amount, slots, surplus, ctx->return_reserved_lo, ctx->return_reserved_hi,
        ratio2digits(factor, UINT32_MAX, &factor_rough, 6), factor,
        ratio2digits(surplus, ctx->return_reserved_hi, &factor_sharp, 12));

  do {
    const size_t left = amount - stored;
    cASSERT0(txn, left > 0 && left <= amount);
    txnid_t id = rkl_turn(&iter, is_lifo(txn));
    if (unlikely(!id)) {
      ERROR("reserve depleted (used %zu slots, left %zu loop %u)", rkl_len(&txn->wr.gc.comeback), left, ctx->loop);
      return MDBX_PROBLEM;
    }
#ifndef _MSC_VER
    MDBX_val key = {.iov_base = &id, .iov_len = sizeof(id)};
#else
    /* avoid MSVC crash and/or ICE */
    MDBX_val key;
    key.iov_base = &id;
    key.iov_len = sizeof(id);
#endif
    MDBX_val data = {.iov_base = nullptr, .iov_len = 0};
    const int err = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err;
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    cASSERT0(txn, data.iov_len >= sizeof(pgno_t) * 2);
    const size_t chunk_hi = data.iov_len / sizeof(pgno_t) - 1;
    cASSERT0(txn, chunk_hi >= 2);
    size_t chunk = left;
    if (chunk > chunk_hi) {
      chunk = chunk_hi;
      const size_t left_slots = rkl_left(&iter, is_lifo(txn));
      if (surplus && left_slots) {
        /* Единственный путь выполнения (набор условий) когда нужно распределять избыток резерва. */
        const size_t hole = (size_t)((chunk_hi * factor) >> 32);
        cASSERT0(txn, hole < chunk_hi && hole <= surplus);
        chunk = chunk_hi - hole;
        cASSERT0(txn, chunk > 0 && chunk <= chunk_hi && surplus >= hole);
        cASSERT0(txn, surplus - hole >= (left + surplus - chunk_hi) * factor >> 32);
        if (chunk > 1 && surplus - hole > (left + surplus - chunk_hi) * factor >> 32)
          chunk -= 1;
      }
      cASSERT0(txn, chunk <= chunk_hi && surplus >= chunk_hi - chunk && chunk <= left);
    }
    surplus -= chunk_hi - chunk;

    pgno_t *const dst = data.iov_base;
    pgno_t *const src = MDBX_PNL_BEGIN(txn->wr.repnl) + left - chunk;
    pgno_t *const from = src, *const to = src + chunk;
    TRACE("%s: fill +%zu (surplus %zu) [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN " (%s)", dbg_prefix(ctx),
          chunk, chunk_hi - chunk, from - txn->wr.repnl, from[0], to - txn->wr.repnl, to[-1], id, "series");
    TRACE("%s: left %zu, surplus %zu, slots %zu", dbg_prefix(ctx), amount - (stored + chunk), surplus,
          rkl_left(&iter, is_lifo(txn)));
    cASSERT0(txn, chunk > 0 && chunk <= chunk_hi && chunk <= left);
    if (unlikely(data.iov_len - gc_chunk_bytes(chunk) >= txn->env->ps)) {
      NOTICE("too long %s-comeback-reserve @%" PRIaTXN ", have %zu bytes, need %zu bytes", "multi", id, data.iov_len,
             gc_chunk_bytes(chunk));
      return MDBX_RESULT_TRUE;
    }

    /* coverity[var_deref_op] */
    *dst = (pgno_t)chunk;
    memcpy(dst + 1, src, chunk * sizeof(pgno_t));
    stored += chunk;
  } while (stored < amount);
  return MDBX_SUCCESS;
}

int gc_update(MDBX_txn *txn, gcu_t *ctx) {
  TRACE("\n>>> @%" PRIaTXN, txn->txnid);
  MDBX_env *const env = txn->env;
  ctx->cursor.next = txn->cursors[FREE_DBI];
  txn->cursors[FREE_DBI] = &ctx->cursor;
  int err;

  if (unlikely(!txn->env->gc.detent))
    txn_gc_detent(txn);

  if (CHECKS2_ENABLED())
    ENSURE_OBJ(txn, audit_ex(txn, 0, false) == MDBX_SUCCESS);

  /* The txn->wr.repnl[] can grow and shrink during this call.
   * The txn->wr.gc.reclaimed[] can grow, then migrate into txn->wr.gc.ready4reuse and later to txn->wr.gc.comeback[].
   * But page numbers cannot disappear from txn->wr.retired_pages[]. */
retry:
  ctx->loop += !(ctx->prev_first_unallocated > txn->geo.first_unallocated);
  TRACE(">> %sstart, loop %u, gc: txn-rkl %zu, detent %" PRIaTXN, (ctx->loop > 1) ? "re" : "", ctx->loop,
        rkl_len(&txn->wr.gc.reclaimed), txn->env->gc.detent);

  tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated));
  tASSERT1(txn, txn_dpl_check(txn));
  if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) {
    ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, ctx->loop);
    err = MDBX_PROBLEM;
    goto bailout;
  }

  if (unlikely(ctx->prev_first_unallocated > txn->geo.first_unallocated)) {
    err = gc_clean_stored_retired(txn, ctx);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;
  }

  ctx->prev_first_unallocated = txn->geo.first_unallocated;
  err = gc_clear_returned(txn, ctx);
  if (unlikely(err != MDBX_SUCCESS))
    goto bailout;

  while (true) {
    /* come back here after each put() in case retired-list changed */
    TRACE("%s", " >> continue");

    tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated));
    err = gc_clear_reclaimed(txn, ctx);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;

    tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated));
    tASSERT1(txn, txn_dpl_check(txn));
    if (CHECKS2_ENABLED())
      ENSURE_OBJ(txn, audit_ex(txn, ctx->retired_stored, false) == MDBX_SUCCESS);

    if (txn->wr.loose_pages) {
      /* merge loose pages into the reclaimed- either retired-list */
      err = gc_merge_loose(txn);
      if (unlikely(err != MDBX_SUCCESS)) {
        if (err == MDBX_RESULT_TRUE)
          continue;
        goto bailout;
      }
      cASSERT0(txn, txn->wr.loose_pages == 0);
    }

    /* return suitable into unallocated space */
    if (txn_refund(txn)) {
      tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
      if (CHECKS2_ENABLED())
        ENSURE_OBJ(txn, audit_ex(txn, ctx->retired_stored, false) == MDBX_SUCCESS);
    }

    if (ctx->retired_stored < pnl_size(txn->wr.retired_pages)) {
      /* store retired-list into GC */
      err = gc_store_retired(txn, ctx);
      if (unlikely(err != MDBX_SUCCESS))
        goto bailout;
      continue;
    }

    tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
    cASSERT0(txn, txn->wr.loose_count == 0);
    if (CHECKS2_ENABLED())
      ENSURE_OBJ(txn, audit_ex(txn, ctx->retired_stored, false) == MDBX_SUCCESS);

    if (unlikely(pnl_size(txn->wr.repnl) + env->maxgc_large1page <= ctx->return_reserved_lo) && !ctx->dense) {
      /* после резервирования было израсходованно слишком много страниц и получилось слишком много резерва */
      TRACE("%s: reclaimed-list %zu < reversed %zu, retry", dbg_prefix(ctx), pnl_size(txn->wr.repnl),
            ctx->return_reserved_lo);
      goto retry;
    }

    if (ctx->return_reserved_hi < pnl_size(txn->wr.repnl)) {
      /* верхней границы резерва НЕ хватает, продолжаем резервирование */
      TRACE(">> %s, %zu...%zu, %s %zu", "reserving", ctx->return_reserved_lo, ctx->return_reserved_hi, "return-left",
            pnl_size(txn->wr.repnl) - ctx->return_reserved_hi);
      err = gc_rerere(txn, ctx);
      if (unlikely(err != MDBX_SUCCESS)) {
        if (err == MDBX_RESULT_TRUE)
          goto retry;
        goto bailout;
      }
      continue;
    }

    if (pnl_size(txn->wr.repnl) > 0) {
      TRACE(">> %s, %s %zu -> %zu...%zu", "filling", "return-reserved", pnl_size(txn->wr.repnl),
            ctx->return_reserved_lo, ctx->return_reserved_hi);
      err = gc_fill_returned(txn, ctx);
      if (unlikely(err != MDBX_SUCCESS)) {
        if (err == MDBX_RESULT_TRUE)
          goto retry;
        goto bailout;
      }
    }
    break;
  }

  cASSERT0(txn, err == MDBX_SUCCESS);
  if (CHECKS2_ENABLED())
    ENSURE_OBJ(txn, audit_ex(txn, ctx->retired_stored + pnl_size(txn->wr.repnl), true) == MDBX_SUCCESS);
  if (unlikely(txn->wr.loose_count > 0)) {
    DEBUG("** restart: got %zu loose pages", txn->wr.loose_count);
    goto retry;
  }

bailout:
  txn->cursors[FREE_DBI] = ctx->cursor.next;

  pnl_clear(txn->wr.repnl);
#if MDBX_ENABLE_PROFGC
  env->lck->pgops.gc_prof.wloops += (uint32_t)ctx->loop;
#endif /* MDBX_ENABLE_PROFGC */
  TRACE("<<< %u loops, rc = %d\n", ctx->loop, err);
  return err;
}

#if MDBX_DEBUG_GCU
#pragma pop_macro("LOG_ENABLED")
#endif /* MDBX_DEBUG_GCU */

static void mdbx_init(void);
static void mdbx_fini(void);

/*----------------------------------------------------------------------------*/
/* mdbx constructor/destructor */

#if defined(_WIN32) || defined(_WIN64)

#if MDBX_BUILD_SHARED_LIBRARY
#if MDBX_WITHOUT_MSVC_CRT && !defined(_DEBUG)
/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks.
 *
 * Define dll's entry point only for Release build when _DEBUG is not defined and
 * MDBX_WITHOUT_MSVC_CRT=ON. if the entry point isn't defined then MSVC's will
 * automatically use DllMainCRTStartup() from CRT library, which also
 * automatically call DllMain() from our mdbx.dll */
#pragma comment(linker, "/ENTRY:DllMain")
#endif /* MDBX_WITHOUT_MSVC_CRT */

BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved)
#else
#if !MDBX_MANUAL_MODULE_HANDLER
static
#endif /* !MDBX_MANUAL_MODULE_HANDLER */
    void NTAPI mdbx_module_handler(PVOID module, DWORD reason, PVOID reserved)
#endif /* MDBX_BUILD_SHARED_LIBRARY */
{
  (void)reserved;
  switch (reason) {
  case DLL_PROCESS_ATTACH:
    windows_import();
    mdbx_init();
    break;
  case DLL_PROCESS_DETACH:
    mdbx_fini();
    break;

  case DLL_THREAD_ATTACH:
    break;
  case DLL_THREAD_DETACH:
    rthc_thread_dtor(module);
    break;
  }
#if MDBX_BUILD_SHARED_LIBRARY
  return TRUE;
#endif
}

#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER
#if defined(_MSC_VER)
#  pragma const_seg(push)
#  pragma data_seg(push)

#  ifndef _M_IX86
     /* kick a linker to create the TLS directory if not already done */
#    pragma comment(linker, "/INCLUDE:_tls_used")
     /* Force some symbol references. */
#    pragma comment(linker, "/INCLUDE:mdbx_tls_anchor")
     /* specific const-segment for WIN64 */
#    pragma const_seg(".CRT$XLB")
     const
#  else
     /* kick a linker to create the TLS directory if not already done */
#    pragma comment(linker, "/INCLUDE:__tls_used")
     /* Force some symbol references. */
#    pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor")
     /* specific data-segment for WIN32 */
#    pragma data_seg(".CRT$XLB")
#  endif

   __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_module_handler;
#  pragma data_seg(pop)
#  pragma const_seg(pop)

#elif defined(__GNUC__)
#  ifndef _M_IX86
     const
#  endif
   PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_module_handler;
#else
#  error FIXME
#endif
#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER */

#else

#if defined(__linux__) || defined(__gnu_linux__)
#include <sys/utsname.h>

MDBX_EXCLUDE_FOR_GPROF
__cold static uint8_t probe_for_WSL(const char *tag) {
  const char *const WSL = strstr(tag, "WSL");
  if (WSL && WSL[3] >= '2' && WSL[3] <= '9')
    return WSL[3] - '0';
  const char *const wsl = strstr(tag, "wsl");
  if (wsl && wsl[3] >= '2' && wsl[3] <= '9')
    return wsl[3] - '0';
  if (WSL || wsl || strcasestr(tag, "Microsoft"))
    /* Expecting no new kernel within WSL1, either it will explicitly
     * marked by an appropriate WSL-version hint. */
    return (globals.linux_kernel_version < /* 4.19.x */ 0x04130000) ? 1 : 2;
  return 0;
}
#endif /* Linux */

#ifdef ENABLE_GPROF
extern void _mcleanup(void);
extern void monstartup(unsigned long, unsigned long);
extern void _init(void);
extern void _fini(void);
extern void __gmon_start__(void) __attribute__((__weak__));
#endif /* ENABLE_GPROF */

MDBX_EXCLUDE_FOR_GPROF
__cold static __attribute__((__constructor__)) void mdbx_global_constructor(void) {
#ifdef ENABLE_GPROF
  if (!&__gmon_start__)
    monstartup((uintptr_t)&_init, (uintptr_t)&_fini);
#endif /* ENABLE_GPROF */

#if defined(__linux__) || defined(__gnu_linux__)
  struct utsname buffer;
  if (uname(&buffer) == 0) {
    int i = 0;
    char *p = buffer.release;
    while (*p && i < 4) {
      if (*p >= '0' && *p <= '9') {
        long number = strtol(p, &p, 10);
        if (number > 0) {
          if (number > 255)
            number = 255;
          globals.linux_kernel_version += number << (24 - i * 8);
        }
        ++i;
      } else {
        ++p;
      }
    }
    /* "Official" way of detecting WSL1 but not WSL2
     * https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364
     *
     * WARNING: False negative detection of WSL1 will result in DATA LOSS!
     * So, the REQUIREMENTS for this code:
     *  1. MUST detect WSL1 without false-negatives.
     *  2. DESIRABLE detect WSL2 but without the risk of violating the first. */
    globals.running_on_WSL1 =
        probe_for_WSL(buffer.version) == 1 || probe_for_WSL(buffer.sysname) == 1 || probe_for_WSL(buffer.release) == 1;
  }
#endif /* Linux */

  mdbx_init();
}

MDBX_EXCLUDE_FOR_GPROF
__cold static __attribute__((__destructor__)) void mdbx_global_destructor(void) {
  mdbx_fini();
#ifdef ENABLE_GPROF
  if (!&__gmon_start__)
    _mcleanup();
#endif /* ENABLE_GPROF */
}

#endif /* ! Windows */

/******************************************************************************/

struct libmdbx_globals globals;

static bool getenv_bool(const char *name, bool default_value) {
  const char *value = osal_getenv(name, false);
  if (value) {
    if (*value == 0 /* implied ON */)
      return true;
    if (strcasecmp(value, "yes") == 0 || strcasecmp(value, "on") == 0 || strcasecmp(value, "true") == 0 ||
        strcasecmp(value, "1") == 0)
      return true;
    if (strcasecmp(value, "no") == 0 || strcasecmp(value, "off") == 0 || strcasecmp(value, "false") == 0 ||
        strcasecmp(value, "0") == 0)
      return false;
  }
  return default_value;
}

__cold static void mdbx_init(void) {
  globals.runtime_flags = (getenv_bool("MDBX_DBG_ASSERT", (MDBX_CHECKING) > 1) ? MDBX_DBG_ASSERT : 0) |
                          (getenv_bool("MDBX_DBG_AUDIT", (MDBX_CHECKING) > 2) ? MDBX_DBG_AUDIT : 0) |
                          (getenv_bool("MDBX_DBG_JITTER", false) ? MDBX_DBG_JITTER : 0) |
                          (getenv_bool("MDBX_DBG_DUMP", false) ? MDBX_DBG_DUMP : 0) |
                          (getenv_bool("MDBX_DBG_LEGACY_MULTIOPEN", false) ? MDBX_DBG_LEGACY_MULTIOPEN : 0) |
                          (getenv_bool("MDBX_DBG_LEGACY_OVERLAP", false) ? MDBX_DBG_LEGACY_OVERLAP : 0) |
                          (getenv_bool("MDBX_DBG_DONT_UPGRADE", false) ? MDBX_DBG_DONT_UPGRADE : 0);
  globals.loglevel = MDBX_LOG_NOTICE;
  ENSURE(osal_fastmutex_init(&globals.debug_lock) == 0);
  osal_ctor();
  ASSERT(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0);
  rthc_ctor();
#if MDBX_CHECKING > 0
  ENSURE(troika_verify_fsm());
  ENSURE(pv2pages_verify());
#endif /* MDBX_CHECKING > 0 */
}

MDBX_EXCLUDE_FOR_GPROF
__cold static void mdbx_fini(void) {
  const mdbx_pid_t current_pid = osal_getpid();
  TRACE(">> pid %zd", (size_t)current_pid);
  rthc_dtor(current_pid);
  osal_dtor();
  TRACE("<< pid %zd\n", (size_t)current_pid);
  ENSURE(osal_fastmutex_destroy(&globals.debug_lock) == 0);
}

/******************************************************************************/

__dll_export
#ifdef __attribute_used__
    __attribute_used__
#elif defined(__GNUC__) || __has_attribute(__used__)
    __attribute__((__used__))
#endif
#ifdef __attribute_externally_visible__
        __attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) ||                            \
    __has_attribute(__externally_visible__)
    __attribute__((__externally_visible__))
#endif
    const struct MDBX_build_info mdbx_build = {
#ifdef MDBX_BUILD_TIMESTAMP
    MDBX_BUILD_TIMESTAMP
#else
    "\"" __DATE__ " " __TIME__ "\""
#endif /* MDBX_BUILD_TIMESTAMP */

    ,
#ifdef MDBX_BUILD_TARGET
    MDBX_BUILD_TARGET
#else
  #if defined(__ANDROID_API__)
    "Android" MDBX_STRINGIFY(__ANDROID_API__)
  #elif defined(__OHOS__)
    "Harmony OS"
  #elif defined(__linux__) || defined(__gnu_linux__)
    "Linux"
  #elif defined(EMSCRIPTEN) || defined(__EMSCRIPTEN__)
    "webassembly"
  #elif defined(__CYGWIN__)
    "CYGWIN"
  #elif defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) \
      || defined(__WINDOWS__)
    "Windows"
  #elif defined(__APPLE__)
    #if (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) \
      || (defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR)
      "iOS"
    #else
      "MacOS"
    #endif
  #elif defined(__FreeBSD__)
    "FreeBSD"
  #elif defined(__DragonFly__)
    "DragonFlyBSD"
  #elif defined(__NetBSD__)
    "NetBSD"
  #elif defined(__OpenBSD__)
    "OpenBSD"
  #elif defined(__bsdi__)
    "UnixBSDI"
  #elif defined(__MACH__)
    "MACH"
  #elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC))
    "HPUX"
  #elif defined(_AIX)
    "AIX"
  #elif defined(__sun) && defined(__SVR4)
    "Solaris"
  #elif defined(__BSD__) || defined(BSD)
    "UnixBSD"
  #elif defined(__unix__) || defined(UNIX) || defined(__unix) \
      || defined(__UNIX) || defined(__UNIX__)
    "UNIX"
  #elif defined(_POSIX_VERSION)
    "POSIX" MDBX_STRINGIFY(_POSIX_VERSION)
  #else
    "UnknownOS"
  #endif /* Target OS */

    "-"

  #if defined(__e2k__) || defined(__elbrus__)
    "Elbrus"
  #elif defined(__amd64__)
    "AMD64"
  #elif defined(__ia32__)
    "IA32"
  #elif defined(__e2k__) || defined(__elbrus__)
    "Elbrus"
  #elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA)
    "Alpha"
  #elif defined(__aarch64__) || defined(_M_ARM64)
    "ARM64"
  #elif defined(__arm__) || defined(__thumb__) || defined(__TARGET_ARCH_ARM) \
      || defined(__TARGET_ARCH_THUMB) || defined(_ARM) || defined(_M_ARM) \
      || defined(_M_ARMT) || defined(__arm)
    "ARM"
  #elif defined(__mips64) || defined(__mips64__) || (defined(__mips) && (__mips >= 64))
    "MIPS64"
  #elif defined(__mips__) || defined(__mips) || defined(_R4000) || defined(__MIPS__)
    "MIPS"
  #elif defined(__hppa64__) || defined(__HPPA64__) || defined(__hppa64)
    "PARISC64"
  #elif defined(__hppa__) || defined(__HPPA__) || defined(__hppa)
    "PARISC"
  #elif defined(__ia64__) || defined(__ia64) || defined(_IA64) \
      || defined(__IA64__) || defined(_M_IA64) || defined(__itanium__)
    "Itanium"
  #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) \
      || defined(__powerpc64) || defined(_ARCH_PPC64)
    "PowerPC64"
  #elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) \
      || defined(__ppc) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__POWERPC__)
    "PowerPC"
  #elif defined(__sparc64__) || defined(__sparc64)
    "SPARC64"
  #elif defined(__sparc__) || defined(__sparc)
    "SPARC"
  #elif defined(__s390__) || defined(__s390) || defined(__zarch__) || defined(__zarch)
    "S390"
  #elif defined(__riscv) || defined(__riscv__) || defined(__RISCV) || defined(__RISCV__)
    "RISC-V (стеклянные бусы)"
  #else
    "UnknownARCH"
  #endif
#endif /* MDBX_BUILD_TARGET */

#ifdef MDBX_BUILD_TYPE
# if defined(_MSC_VER)
#   pragma message("Configuration-depended MDBX_BUILD_TYPE: " MDBX_BUILD_TYPE)
# endif
    "-" MDBX_BUILD_TYPE
#endif /* MDBX_BUILD_TYPE */
    ,
    "DEBUG=" MDBX_STRINGIFY(MDBX_DEBUG)
    " CHECKING=" MDBX_STRINGIFY(MDBX_CHECKING)
#ifdef ENABLE_GPROF
    " ENABLE_GPROF"
#endif /* ENABLE_GPROF */
    " WORDBITS=" MDBX_STRINGIFY(MDBX_WORDBITS)
    " BYTE_ORDER="
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
    "LE"
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
    "BE"
#else
    #error "FIXME: Unsupported byte order"
#endif /* __BYTE_ORDER__ */
    " BIGFOOT=" MDBX_STRINGIFY(MDBX_ENABLE_BIGFOOT)
    " ENV_CHECKPID=" MDBX_ENV_CHECKPID_CONFIG
    " TXN_CHECKOWNER=" MDBX_TXN_CHECKOWNER_CONFIG
    " 64BIT_ATOMIC=" MDBX_64BIT_ATOMIC_CONFIG
    " 64BIT_CAS=" MDBX_64BIT_CAS_CONFIG
    " TRUST_RTC=" MDBX_TRUST_RTC_CONFIG
    " AVOID_MSYNC=" MDBX_STRINGIFY(MDBX_AVOID_MSYNC)
    " REFUND=" MDBX_STRINGIFY(MDBX_ENABLE_REFUND)
    " MINCORE=" MDBX_STRINGIFY(MDBX_USE_MINCORE)
    " PGOP_STAT=" MDBX_STRINGIFY(MDBX_ENABLE_PGOP_STAT)
    " PROFGC=" MDBX_STRINGIFY(MDBX_ENABLE_PROFGC)
    " PGET_STAT=" MDBX_STRINGIFY(MDBX_ENABLE_PGET_STAT)
#if MDBX_DISABLE_VALIDATION
    " DISABLE_VALIDATION=YES"
#endif /* MDBX_DISABLE_VALIDATION */
#ifdef __SANITIZE_ADDRESS__
    " SANITIZE_ADDRESS=YES"
#endif /* __SANITIZE_ADDRESS__ */
#ifdef ENABLE_MEMCHECK
    " MEMCHECK=YES"
#endif /* ENABLE_MEMCHECK */
#ifdef ENABLE_SYSTEMTAP
    " SYSTEMTAP=YES"
#elif defined(ENABLE_DTRACE)
    " DTRACE=YES"
#endif /* ENABLE_DTRACE || ENABLE_SYSTEMTAP */
#ifdef _GNU_SOURCE
    " _GNU_SOURCE=YES"
#else
    " _GNU_SOURCE=NO"
#endif /* _GNU_SOURCE */
#ifdef __APPLE__
    " MDBX_APPLE_SPEED_INSTEADOF_DURABILITY=" MDBX_STRINGIFY(MDBX_APPLE_SPEED_INSTEADOF_DURABILITY)
#endif /* MacOS */
#if defined(_WIN32) || defined(_WIN64)
    " WITHOUT_MSVC_CRT=" MDBX_STRINGIFY(MDBX_WITHOUT_MSVC_CRT)
    " BUILD_SHARED_LIBRARY=" MDBX_STRINGIFY(MDBX_BUILD_SHARED_LIBRARY)
#if !MDBX_BUILD_SHARED_LIBRARY
    " MANUAL_MODULE_HANDLER=" MDBX_STRINGIFY(MDBX_MANUAL_MODULE_HANDLER)
#endif
    " WINVER=" MDBX_STRINGIFY(WINVER)
#else /* Windows */
    " LOCKING=" MDBX_LOCKING_CONFIG
    " OFDLOCKS=" MDBX_USE_OFDLOCKS_CONFIG
    " FALLOCATE=" MDBX_USE_FALLOCATE_CONFIG
#endif /* !Windows */
    " CACHELINE_SIZE=" MDBX_STRINGIFY(MDBX_CACHELINE_SIZE)
    " CPU_WRITEBACK_INCOHERENT=" MDBX_STRINGIFY(MDBX_CPU_WRITEBACK_INCOHERENT)
    " MMAP_INCOHERENT_CPU_CACHE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_CPU_CACHE)
    " MMAP_INCOHERENT_FILE_WRITE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_FILE_WRITE)
    " UNALIGNED_OK=" MDBX_STRINGIFY(MDBX_UNALIGNED_OK)
#if MDBX_PNL_ASCENDING
    " PNL_ASCENDING=" MDBX_STRINGIFY(MDBX_PNL_ASCENDING)
#endif /* MDBX_PNL_ASCENDING */
    ,
#ifdef MDBX_BUILD_COMPILER
    MDBX_BUILD_COMPILER
#else
  #ifdef __INTEL_COMPILER
    "Intel C/C++ " MDBX_STRINGIFY(__INTEL_COMPILER)
  #elif defined(__apple_build_version__)
    "Apple clang " MDBX_STRINGIFY(__apple_build_version__)
  #elif defined(__ibmxl__)
    "IBM clang C " MDBX_STRINGIFY(__ibmxl_version__) "." MDBX_STRINGIFY(__ibmxl_release__)
    "." MDBX_STRINGIFY(__ibmxl_modification__) "." MDBX_STRINGIFY(__ibmxl_ptf_fix_level__)
  #elif defined(__clang__)
    "clang " MDBX_STRINGIFY(__clang_version__)
  #elif defined(__MINGW64__)
    "MINGW-64 " MDBX_STRINGIFY(__MINGW64_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW64_MINOR_VERSION)
  #elif defined(__MINGW32__)
    "MINGW-32 " MDBX_STRINGIFY(__MINGW32_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW32_MINOR_VERSION)
  #elif defined(__MINGW__)
    "MINGW " MDBX_STRINGIFY(__MINGW_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW_MINOR_VERSION)
  #elif defined(__IBMC__)
    "IBM C " MDBX_STRINGIFY(__IBMC__)
  #elif defined(__GNUC__)
    "GNU C/C++ "
    #ifdef __VERSION__
      __VERSION__
    #else
      MDBX_STRINGIFY(__GNUC__) "." MDBX_STRINGIFY(__GNUC_MINOR__) "." MDBX_STRINGIFY(__GNUC_PATCHLEVEL__)
    #endif
  #elif defined(_MSC_VER)
    "MSVC " MDBX_STRINGIFY(_MSC_FULL_VER) "-" MDBX_STRINGIFY(_MSC_BUILD)
  #else
    "Unknown compiler"
  #endif
#endif /* MDBX_BUILD_COMPILER */
    ,
#ifdef MDBX_BUILD_FLAGS_CONFIG
    MDBX_BUILD_FLAGS_CONFIG
#endif /* MDBX_BUILD_FLAGS_CONFIG */
#if defined(MDBX_BUILD_FLAGS_CONFIG) && defined(MDBX_BUILD_FLAGS)
    " "
#endif
#ifdef MDBX_BUILD_FLAGS
    MDBX_BUILD_FLAGS
#endif /* MDBX_BUILD_FLAGS */
#if !(defined(MDBX_BUILD_FLAGS_CONFIG) || defined(MDBX_BUILD_FLAGS))
    "undefined (please use correct build script)"
#ifdef _MSC_VER
#pragma message("warning: Build flags undefined. Please use correct build script")
#else
#warning "Build flags undefined. Please use correct build script"
#endif // _MSC_VER
#endif
  , MDBX_BUILD_METADATA
};

#ifdef __SANITIZE_ADDRESS__
#if !defined(_MSC_VER) || __has_attribute(weak)
LIBMDBX_API __attribute__((__weak__))
#endif
const char *__asan_default_options(void) {
  return "symbolize=1:allow_addr2line=1:"
#if MDBX_DEBUG > 0
         "debug=1:"
         "verbosity=2:"
#endif /* MDBX_DEBUG */
         "log_threads=1:"
         "report_globals=1:"
         "replace_str=1:replace_intrin=1:"
         "malloc_context_size=9:"
#if !defined(__APPLE__)
         "detect_leaks=1:"
#endif
         "check_printf=1:"
         "detect_deadlocks=1:"
#ifndef LTO_ENABLED
         "check_initialization_order=1:"
#endif
         "detect_stack_use_after_return=1:"
         "intercept_tls_get_addr=1:"
         "decorate_proc_maps=1:"
         "abort_on_error=1";
}
#endif /* __SANITIZE_ADDRESS__ */

static bool histogram_check(const struct MDBX_chk_histogram *p, size_t adj) {
  size_t quantiry = p->le1_count;
  for (size_t i = 0; i < ARRAY_LENGTH(p->ranges); ++i)
    quantiry += p->ranges[i].count;

  return quantiry == p->count - adj;
}

static void histogram_reduce_move(struct MDBX_chk_histogram *p, size_t point) {
  ASSERT(histogram_check(p, 1));
  // объединяем
  p->ranges[point].end = p->ranges[point + 1].end;
  p->ranges[point].amount += p->ranges[point + 1].amount;
  p->ranges[point].count += p->ranges[point + 1].count;

  // перемещаем хвост
  while (++point < ARRAY_LENGTH(p->ranges) - 1)
    p->ranges[point] = p->ranges[point + 1];

  // обнуляем последний элемент и продолжаем
  p->ranges[ARRAY_LENGTH(p->ranges) - 1].count = 0;
  ASSERT(histogram_check(p, 1));
}

static __hot void histogram_put(const size_t v, struct MDBX_chk_histogram *p,
                                intptr_t (*finder)(struct MDBX_chk_histogram *p, size_t v), unsigned options) {
  STATIC_ASSERT(ARRAY_LENGTH(p->ranges) > 2);
  p->amount += v;
  p->count += 1;
  if (v < ((options & HISTOGRAM_LE0) ? 1u : 2u)) {
    p->le1_amount += v;
    p->le1_count += 1;
    return;
  }

  const size_t size = ARRAY_LENGTH(p->ranges), last = size - 1;
  for (;;) {
    size_t i = 0;
    while (i < size && p->ranges[i].count && v >= p->ranges[i].begin) {
      if (v < p->ranges[i].end) {
        // значение попадает в существующий интервал
        p->ranges[i].amount += v;
        p->ranges[i].count += 1;
        return;
      }
      ++i;
    }
    if (p->ranges[last].count == 0) {
      // использованы еще не все слоты, добавляем интервал
      ASSERT(i < size && histogram_check(p, 1));
      if (p->ranges[i].count && i < last) {
        // раздвигаем
        memmove(p->ranges + i + 1, p->ranges + i, (last - i) * sizeof(p->ranges[0]));
      }
      p->ranges[i].begin = v;
      p->ranges[i].end = v + 1;
      p->ranges[i].amount = v;
      p->ranges[i].count = 1;
      ASSERT(histogram_check(p, 0));
      return;
    }

    const intptr_t point = finder(p, v);
    if (point >= 0)
      // ошибка меньше если слить соседние слоты и добавить новый
      histogram_reduce_move(p, point);
    else {
      // ошибка меньше если расширить слот и добавить к нему
      i = -point - 1;
      p->ranges[i].begin = (v < p->ranges[i].begin) ? v : p->ranges[i].begin;
      p->ranges[i].end = (v < p->ranges[i].end) ? p->ranges[i].end : v + 1;
      p->ranges[i].amount += v;
      p->ranges[i].count += 1;
      return;
    }
  }
}

//------------------------------------------------------------------------------

__hot static intptr_t histogram_minimize_error(struct MDBX_chk_histogram *p, size_t v) {
#if MDBX_HISTOGRAM_USING_128BIT
  bin128_t best_reduce = {.l = UINT64_MAX, .h = UINT64_MAX}, best_enhance = best_reduce;
#else
  uint64_t best_reduce = UINT64_MAX, best_enhance = best_reduce;
#endif /* MDBX_HISTOGRAM_USING_128BIT */

  // ищем пару для слияния с минимальной ошибкой
  intptr_t reduce = 0;
  for (size_t i = 0; i < ARRAY_LENGTH(p->ranges) - 1; ++i) {
    const size_t b1 = p->ranges[i].begin, e1 = p->ranges[i].end;
    const size_t b2 = p->ranges[i + 1].begin, e2 = p->ranges[i + 1].end;
    const uint64_t n1 = p->ranges[i].count, n2 = p->ranges[i + 1].count;
    ASSERT(n1 > 0 && b1 > 0 && b1 < e1);
    ASSERT(n2 > 0 && b2 > 0 && b2 < e2);
    ASSERT(e1 <= b2);
    // за ошибку принимаем площадь изменений на гистограмме при слиянии слотов
    // s1 = (l1 = e1 - b1) * n1; s2 = (l2 = e2 - b2) * n2
    // sx = (lx = e2 - b1) * (nx = n1 + n2) == e2*n1 + e2*n2 - b1*n1 - b1*n2
    // err = s1 + s2 - sx == e1*n1 - b1*n1 + e2*n2 - b2*n2 - e2*n1 - e2*n2 + b1*n1 + b1*n
    // err = n1 * (e2 - e1) + n2 * (b2 - b1)
#if MDBX_HISTOGRAM_USING_128BIT
    const bin128_t err = u128_add(mul64x64_128(e2 - e1, n1), mul64x64_128(b2 - b1, n2));
    if (u128_gt(err, best_reduce))
      continue;
#else
    const uint64_t err = (e2 - e1) * n1 + (b2 - b1) * n2;
    if (err > best_reduce)
      continue;
#endif /* MDBX_HISTOGRAM_USING_128BIT */

    reduce = i;
    best_reduce = err;
  }

  // ищем слот для расширения с минимальной ошибкой
  intptr_t enhance = 0;
  for (size_t i = 0; i < ARRAY_LENGTH(p->ranges); ++i) {
    const size_t b = p->ranges[i].begin, e = p->ranges[i].end;
    const uint64_t n = p->ranges[i].count;
    const size_t ve = (v < e) ? e : v + 1;
    ASSERT(n > 0 && b > 0 && b < e);
    if (i > 0 && v < p->ranges[i - 1].end)
      // пропускаем если расширение этого слота приведёт к пересечению с предыдущим
      continue;
    if (i < ARRAY_LENGTH(p->ranges) - 1 && v >= p->ranges[i + 1].begin)
      // пропускаем если расширение этого слота приведёт к пересечению со следующим
      continue;

    // за ошибку принимаем площадь изменений на гистограмме при расширении слота
#if MDBX_HISTOGRAM_USING_128BIT
    const bin128_t err =
        u128_add((v < b) ? mul64x64_128(b - v, n) : u128(v - b), (ve < e) ? u128(e - ve) : mul64x64_128(ve - e, n));
    if (u128_gt(err, best_enhance))
      continue;
#else
    const uint64_t err = ((v < b) ? (b - v) * n : v - b) + ((ve < e) ? e - ve : (ve - e) * n);
    if (err > best_enhance)
      continue;
#endif /* MDBX_HISTOGRAM_USING_128BIT */

    enhance = i;
    best_enhance = err;
  }

#if MDBX_HISTOGRAM_USING_128BIT
  if (u128_gt(best_enhance, best_reduce))
    return reduce;
#else
  if (best_enhance > best_reduce)
    return reduce;
#endif

  return -(enhance + 1);
}

__hot void histogram_acc_ex(const size_t v, struct MDBX_chk_histogram *p, unsigned options) {
  histogram_put(v, p, histogram_minimize_error, options);
}

//------------------------------------------------------------------------------

__cold MDBX_chk_line_t *histogram_dist(MDBX_chk_line_t *line, const struct MDBX_chk_histogram *histogram,
                                       const char *prefix, const char *first, bool amount) {
  /* https://en.wikipedia.org/wiki/Multiplication_sign */
#if defined(unix) || defined(linux) || defined(__unix__) || defined(__unix) || defined(__linux__) ||                   \
    defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE)
#define UNICODE_MULSIGN_STR "×"
#define UNICODE_MULSIGN_FMT "s"
#elif defined(_WIN32) || defined(_WIN64)
#define UNICODE_MULSIGN_STR L"\u00d7"
#define UNICODE_MULSIGN_FMT "ls"
#else
#define UNICODE_MULSIGN_STR "*"
#define UNICODE_MULSIGN_FMT "s"
#endif
  if (histogram->count) {
    line = chk_print(line, "%s:", prefix);
    const char *comma = "";
    const size_t first_val = amount ? histogram->le1_amount : histogram->le1_count;
    if (first_val) {
      chk_print(line, " %s%" UNICODE_MULSIGN_FMT "%" PRIuSIZE, first, UNICODE_MULSIGN_STR, first_val);
      comma = ",";
    }
    for (size_t n = 0; n < ARRAY_LENGTH(histogram->ranges); ++n)
      if (histogram->ranges[n].count) {
        chk_print(line, "%s %" PRIuSIZE, comma, histogram->ranges[n].begin);
        if (histogram->ranges[n].begin != histogram->ranges[n].end - 1)
          chk_print(line, "-%" PRIuSIZE, histogram->ranges[n].end - 1);
        line = chk_print(line, "%" UNICODE_MULSIGN_FMT "%" PRIuSIZE, UNICODE_MULSIGN_STR,
                         amount ? histogram->ranges[n].amount : histogram->ranges[n].count);
        comma = ",";
      }

    ENSURE_MSG(histogram_check(histogram, 0), "Historgam related bug, please report this");
  }
  return line;
}

__cold MDBX_chk_line_t *histogram_print(MDBX_chk_scope_t *scope, MDBX_chk_line_t *line,
                                        const struct MDBX_chk_histogram *histogram, const char *prefix,
                                        const char *first, bool amount) {
  if (histogram->count) {
    line = chk_print(line, "%s %" PRIuSIZE, prefix, amount ? histogram->amount : histogram->count);
    if (scope->verbosity > MDBX_chk_info)
      line = chk_puts(histogram_dist(line, histogram, " (distribution", first, amount), ")");
  }
  return line;
}

#if !(defined(_WIN32) || defined(_WIN64))
/*----------------------------------------------------------------------------*
 * POSIX/non-Windows LCK-implementation */

#if MDBX_LOCKING == MDBX_LOCKING_SYSV
#include <sys/sem.h>
#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */

/* Описание реализации блокировок для POSIX & Linux:
 *
 * lck-файл отображается в память, в нём организуется таблица читателей и
 * размещаются совместно используемые posix-мьютексы (futex). Посредством
 * этих мьютексов (см struct lck_t) реализуются:
 *  - Блокировка таблицы читателей для регистрации,
 *    т.е. функции lck_rdt_lock() и lck_rdt_unlock().
 *  - Блокировка БД для пишущих транзакций,
 *    т.е. функции lck_txn_lock() и lck_txn_unlock().
 *
 * Остальной функционал реализуется отдельно посредством файловых блокировок:
 *  - Первоначальный захват БД в режиме exclusive/shared и последующий перевод
 *    в операционный режим, функции lck_seize() и lck_downgrade().
 *  - Проверка присутствие процессов-читателей,
 *    т.е. функции lck_rpid_set(), lck_rpid_clear() и lck_rpid_check().
 *
 * Для блокировки файлов используется fcntl(F_SETLK), так как:
 *  - lockf() оперирует только эксклюзивной блокировкой и требует
 *    открытия файла в RW-режиме.
 *  - flock() не гарантирует атомарности при смене блокировок
 *    и оперирует только всем файлом целиком.
 *  - Для контроля процессов-читателей используются однобайтовые
 *    range-блокировки lck-файла посредством fcntl(F_SETLK). При этом
 *    в качестве позиции используется pid процесса-читателя.
 *  - Для первоначального захвата и shared/exclusive выполняется блокировка
 *    основного файла БД и при успехе lck-файла.
 *
 * ----------------------------------------------------------------------------
 * УДЕРЖИВАЕМЫЕ БЛОКИРОВКИ В ЗАВИСИМОСТИ ОТ РЕЖИМА И СОСТОЯНИЯ
 *
 * Эксклюзивный режим без lck-файла:
 *   = заблокирован весь dxb-файл посредством F_RDLCK или F_WRLCK,
 *     в зависимости от MDBX_RDONLY.
 *
 * Не-операционный режим на время пере-инициализации и разрушении lck-файла:
 *   = F_WRLCK блокировка первого байта lck-файла, другие процессы ждут её
 *     снятия при получении F_RDLCK через F_SETLKW.
 *   - блокировки dxb-файла могут меняться до снятие эксклюзивной блокировки
 *     lck-файла:
 *       + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле
 *         посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
 *       + для ЭКСКЛЮЗИВНОГО режима блокировка всего dxb-файла
 *         посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
 *
 * ОПЕРАЦИОННЫЙ режим с lck-файлом:
 *   = F_RDLCK блокировка первого байта lck-файла, другие процессы не могут
 *     получить F_WRLCK и таким образом видят что БД используется.
 *   + F_WRLCK блокировка pid-байта в clk-файле после первой транзакции чтения.
 *   + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле
 *     посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
 *   + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла
 *     посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
 */

#if MDBX_USE_OFDLOCKS
static int op_setlk, op_setlkw, op_getlk;
__cold static void choice_fcntl(void) {
  ASSERT(!op_setlk && !op_setlkw && !op_getlk);
  if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0
#if defined(__linux__) || defined(__gnu_linux__)
      && globals.linux_kernel_version > 0x030f0000 /* OFD locks are available since 3.15, but engages here
                                                      only for 3.16 and later kernels (i.e. LTS) because
                                                      of reliability reasons */
#endif                                             /* linux */
  ) {
    op_setlk = MDBX_F_OFD_SETLK;
    op_setlkw = MDBX_F_OFD_SETLKW;
    op_getlk = MDBX_F_OFD_GETLK;
    return;
  }
  op_setlk = MDBX_F_SETLK;
  op_setlkw = MDBX_F_SETLKW;
  op_getlk = MDBX_F_GETLK;
}
#else
#define op_setlk MDBX_F_SETLK
#define op_setlkw MDBX_F_SETLKW
#define op_getlk MDBX_F_GETLK
#endif /* MDBX_USE_OFDLOCKS */

static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck, const off_t offset, off_t len) {
  STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) && sizeof(off_t) >= sizeof(size_t));
#if defined(__ANDROID_API__) && __ANDROID_API__ < 24
  STATIC_ASSERT_MSG((sizeof(off_t) * CHAR_BIT == MDBX_WORDBITS),
                    "The bitness of system `off_t` type is mismatch. Please "
                    "fix build and/or NDK configuration.");
#endif /* Android && API < 24 */
  ASSERT(offset >= 0 && len > 0);
  ASSERT((uint64_t)offset < (uint64_t)INT64_MAX && (uint64_t)len < (uint64_t)INT64_MAX &&
         (uint64_t)(offset + len) > (uint64_t)offset);

  ASSERT((uint64_t)offset < (uint64_t)OFF_T_MAX && (uint64_t)len <= (uint64_t)OFF_T_MAX &&
         (uint64_t)(offset + len) <= (uint64_t)OFF_T_MAX);

  ASSERT((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) == ((uint64_t)offset + (uint64_t)len));

  jitter4testing(true);
  for (;;) {
    MDBX_STRUCT_FLOCK lock_op = {.l_type = lck, .l_whence = SEEK_SET, .l_start = offset, .l_len = len};
    STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) && sizeof(off_t) <= sizeof(lock_op.l_len) &&
                          OFF_T_MAX == (off_t)OFF_T_MAX,
                      "Support for large/64-bit-sized files is misconfigured "
                      "for the target system and/or toolchain. "
                      "Please fix it or at least disable it completely.");
    int rc = MDBX_FCNTL(fd, cmd, &lock_op);
    jitter4testing(true);
    if (rc != -1) {
      if (cmd == op_getlk) {
        /* Checks reader by pid. Returns:
         *   MDBX_RESULT_TRUE   - if pid is live (reader holds a lock).
         *   MDBX_RESULT_FALSE  - if pid is dead (a lock could be placed). */
        return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
      }
      return MDBX_SUCCESS;
    }
    rc = errno;
#if MDBX_USE_OFDLOCKS
    if (ignore_enosys_and_einval(rc) == MDBX_RESULT_TRUE &&
        (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW || cmd == MDBX_F_OFD_GETLK)) {
      /* fallback to non-OFD locks */
      if (cmd == MDBX_F_OFD_SETLK)
        cmd = MDBX_F_SETLK;
      else if (cmd == MDBX_F_OFD_SETLKW)
        cmd = MDBX_F_SETLKW;
      else
        cmd = MDBX_F_GETLK;
      op_setlk = MDBX_F_SETLK;
      op_setlkw = MDBX_F_SETLKW;
      op_getlk = MDBX_F_GETLK;
      continue;
    }
#endif /* MDBX_USE_OFDLOCKS */
    if (rc != EINTR || cmd == op_setlkw) {
      ASSERT(MDBX_IS_ERROR(rc));
      return rc;
    }
  }
}

static int lck_setlk_with3retries(const mdbx_filehandle_t fd, const int lck, const off_t offset, off_t len) {
  ASSERT(lck != F_UNLCK);
  int retry_left = 3;
#if defined(__ANDROID_API__)
  retry_left *= 3;
#endif /* Android */
  while (true) {
    int rc = lck_op(fd, op_setlk, lck, offset, len);
    if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK) || --retry_left < 1)
      return rc;
#if defined(__ANDROID_API__)
    if (retry_left == 5 || retry_left == 3) {
      usleep(1000 * 42 / 2);
      continue;
    }
#endif /* Android */
    if (osal_yield())
      return rc;
  }
}

int osal_lockfile(mdbx_filehandle_t fd, bool wait) {
#if MDBX_USE_OFDLOCKS
  if (unlikely(op_setlk == 0))
    choice_fcntl();
#endif /* MDBX_USE_OFDLOCKS */
  return lck_op(fd, wait ? op_setlkw : op_setlk, F_WRLCK, 0, OFF_T_MAX);
}

int lck_rpid_set(MDBX_env *env) {
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);
  ASSERT(env->pid > 0);
  if (unlikely(osal_getpid() != env->pid))
    return MDBX_PANIC;
  return lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, env->pid, 1);
}

int lck_rpid_clear(MDBX_env *env) {
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);
  ASSERT(env->pid > 0);
  return lck_op(env->lck_mmap.fd, op_setlk, F_UNLCK, env->pid, 1);
}

int lck_rpid_check(MDBX_env *env, uint32_t pid) {
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);
  ASSERT(pid > 0);
  return lck_op(env->lck_mmap.fd, op_getlk, F_WRLCK, pid, 1);
}

/*---------------------------------------------------------------------------*/

#if MDBX_LOCKING > MDBX_LOCKING_SYSV
int lck_ipclock_stubinit(osal_ipclock_t *ipc) {
#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
  return sem_init(ipc, false, 1) ? errno : 0;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
  return pthread_mutex_init(ipc, nullptr);
#else
#error "FIXME"
#endif
}

int lck_ipclock_destroy(osal_ipclock_t *ipc) {
#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
  return sem_destroy(ipc) ? errno : 0;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
  return pthread_mutex_destroy(ipc);
#else
#error "FIXME"
#endif
}
#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */

static int check_fstat(MDBX_env *env) {
  struct stat st;

  int rc = MDBX_SUCCESS;
  if (fstat(env->lazy_fd, &st)) {
    rc = errno;
    ERROR("fstat(%s), err %d", "DXB", rc);
    return rc;
  }

  if (!S_ISREG(st.st_mode) || st.st_nlink < 1) {
#ifdef EBADFD
    rc = EBADFD;
#else
    rc = EPERM;
#endif
    ERROR("%s %s, err %d", "DXB", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc);
    return rc;
  }

  if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) {
    VERBOSE("dxb-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size);
    rc = MDBX_RESULT_TRUE;
  }

  //----------------------------------------------------------------------------

  if (fstat(env->lck_mmap.fd, &st)) {
    rc = errno;
    ERROR("fstat(%s), err %d", "LCK", rc);
    return rc;
  }

  if (!S_ISREG(st.st_mode) || st.st_nlink < 1) {
#ifdef EBADFD
    rc = EBADFD;
#else
    rc = EPERM;
#endif
    ERROR("%s %s, err %d", "LCK", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc);
    return rc;
  }

  /* Checking file size for detect the situation when we got the shared lock
   * immediately after lck_destroy(). */
  if (st.st_size < (off_t)(sizeof(lck_t) + sizeof(reader_slot_t))) {
    VERBOSE("lck-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size);
    rc = MDBX_RESULT_TRUE;
  }

  return rc;
}

__cold int lck_seize(MDBX_env *env) {
  ASSERT(env->lazy_fd != INVALID_HANDLE_VALUE);
  if (unlikely(osal_getpid() != env->pid))
    return MDBX_PANIC;

  int rc = MDBX_SUCCESS;
#if defined(__linux__) || defined(__gnu_linux__)
  if (unlikely(globals.running_on_WSL1)) {
    rc = ENOLCK /* No record locks available */;
    ERROR("%s, err %u",
          "WSL1 (Windows Subsystem for Linux) is mad and trouble-full, "
          "injecting failure to avoid data loss",
          rc);
    return rc;
  }
#endif /* Linux */

#if MDBX_USE_OFDLOCKS
  if (unlikely(op_setlk == 0))
    choice_fcntl();
#endif /* MDBX_USE_OFDLOCKS */

  if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) {
    /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
    rc = lck_setlk_with3retries(env->lazy_fd, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
    if (rc != MDBX_SUCCESS) {
      ERROR("%s, err %u", "without-lck", rc);
      eASSERT0(env, MDBX_IS_ERROR(rc));
      return rc;
    }
    return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
  }

retry:
  if (rc == MDBX_RESULT_TRUE) {
    rc = lck_op(env->lck_mmap.fd, op_setlk, F_UNLCK, 0, 1);
    if (rc != MDBX_SUCCESS) {
      ERROR("%s, err %u", "unlock-before-retry", rc);
      eASSERT0(env, MDBX_IS_ERROR(rc));
      return rc;
    }
  }

  /* Firstly try to get exclusive locking.  */
  rc = lck_setlk_with3retries(env->lck_mmap.fd, F_WRLCK, 0, 1);
  if (rc == MDBX_SUCCESS) {
    rc = check_fstat(env);
    if (MDBX_IS_ERROR(rc))
      return rc;

  continue_dxb_exclusive:
    rc = lck_setlk_with3retries(env->lazy_fd, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
    if (rc == MDBX_SUCCESS)
      return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;

    int err = check_fstat(env);
    if (MDBX_IS_ERROR(err))
      return err;

    /* the cause may be a collision with POSIX's file-lock recovery. */
    if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) {
      ERROR("%s, err %u", "dxb-exclusive", rc);
      eASSERT0(env, MDBX_IS_ERROR(rc));
      return rc;
    }

    /* Fallback to lck-shared */
  } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) {
    ERROR("%s, err %u", "try-exclusive", rc);
    eASSERT0(env, MDBX_IS_ERROR(rc));
    return rc;
  }

  /* Here could be one of two:
   *  - lck_destroy() from the another process was hold the lock
   *    during a destruction.
   *  - either lck_seize() from the another process was got the exclusive
   *    lock and doing initialization.
   * For distinguish these cases will use size of the lck-file later. */

  /* Wait for lck-shared now. */
  /* Here may be await during transient processes, for instance until another
   * competing process doesn't call lck_downgrade(). */
  rc = lck_op(env->lck_mmap.fd, op_setlkw, F_RDLCK, 0, 1);
  if (rc != MDBX_SUCCESS) {
    ERROR("%s, err %u", "try-shared", rc);
    eASSERT0(env, MDBX_IS_ERROR(rc));
    return rc;
  }

  rc = check_fstat(env);
  if (rc == MDBX_RESULT_TRUE)
    goto retry;
  if (rc != MDBX_SUCCESS) {
    ERROR("%s, err %u", "lck_fstat", rc);
    return rc;
  }

  /* got shared, retry exclusive */
  rc = lck_setlk_with3retries(env->lck_mmap.fd, F_WRLCK, 0, 1);
  if (rc == MDBX_SUCCESS)
    goto continue_dxb_exclusive;

  if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) {
    ERROR("%s, err %u", "try-exclusive", rc);
    eASSERT0(env, MDBX_IS_ERROR(rc));
    return rc;
  }

  /* Lock against another process operating in without-lck or exclusive mode. */
  rc = lck_setlk_with3retries(env->lazy_fd, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->pid, 1);
  if (rc != MDBX_SUCCESS) {
    ERROR("%s, err %u", "lock-against-without-lck", rc);
    eASSERT0(env, MDBX_IS_ERROR(rc));
    return rc;
  }

  /* Done: return with shared locking. */
  return MDBX_RESULT_FALSE;
}

int lck_downgrade(MDBX_env *env) {
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);
  if (unlikely(osal_getpid() != env->pid))
    return MDBX_PANIC;

  int rc = MDBX_SUCCESS;
  if ((env->flags & MDBX_EXCLUSIVE) == 0) {
    rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid);
    if (rc == MDBX_SUCCESS)
      rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, env->pid + 1, OFF_T_MAX - env->pid - 1);
  }
  if (rc == MDBX_SUCCESS)
    rc = lck_setlk_with3retries(env->lck_mmap.fd, F_RDLCK, 0, 1);
  if (unlikely(rc != 0)) {
    ERROR("%s, err %u", "lck", rc);
    ASSERT(MDBX_IS_ERROR(rc));
  }
  return rc;
}

int lck_upgrade(MDBX_env *env, bool dont_wait) {
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);
  if (unlikely(osal_getpid() != env->pid))
    return MDBX_PANIC;

  const int cmd = dont_wait ? op_setlk : op_setlkw;
  int rc = lck_op(env->lck_mmap.fd, cmd, F_WRLCK, 0, 1);
  if (rc == MDBX_SUCCESS && (env->flags & MDBX_EXCLUSIVE) == 0) {
    rc = (env->pid > 1) ? lck_op(env->lazy_fd, cmd, F_WRLCK, 0, env->pid - 1) : MDBX_SUCCESS;
    if (rc == MDBX_SUCCESS) {
      rc = lck_op(env->lazy_fd, cmd, F_WRLCK, env->pid + 1, OFF_T_MAX - env->pid - 1);
      if (rc != MDBX_SUCCESS && env->pid > 1 && lck_setlk_with3retries(env->lazy_fd, F_UNLCK, 0, env->pid - 1))
        rc = MDBX_PANIC;
    }
    if (rc != MDBX_SUCCESS && lck_setlk_with3retries(env->lck_mmap.fd, F_RDLCK, 0, 1))
      rc = MDBX_PANIC;
  }
  if (unlikely(rc != 0)) {
    ERROR("%s, err %u", "lck", rc);
    ASSERT(MDBX_IS_ERROR(rc));
  }
  return rc;
}

__cold int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const mdbx_pid_t current_pid) {
  eASSERT0(env, osal_getpid() == current_pid);
  int rc = MDBX_SUCCESS;
  struct stat lck_info;
  lck_t *lck = env->lck;
  if (lck && lck == env->lck_mmap.lck && !inprocess_neighbor &&
      /* try get exclusive access */
      lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 &&
      /* if LCK was not removed */
      fstat(env->lck_mmap.fd, &lck_info) == 0 && lck_info.st_nlink > 0 &&
      lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX) == 0) {

    VERBOSE("%p got exclusive, drown ipc-locks", __Wpedantic_format_voidptr(env));
    eASSERT0(env, current_pid == env->pid);
#if MDBX_LOCKING == MDBX_LOCKING_SYSV
    if (env->me_sysv_ipc.semid != -1)
      rc = semctl(env->me_sysv_ipc.semid, 2, IPC_RMID) ? errno : 0;
#else
    rc = lck_ipclock_destroy(&lck->rdt_lock);
    if (rc == 0)
      rc = lck_ipclock_destroy(&lck->wrt_lock);
#endif /* MDBX_LOCKING */

    eASSERT0(env, rc == 0);
    if (rc == 0) {
      const bool synced = lck->unsynced_pages.weak == 0;
      osal_munmap(&env->lck_mmap);
      if (synced && env->lck_mmap.fd != INVALID_HANDLE_VALUE)
        rc = ftruncate(env->lck_mmap.fd, 0) ? errno : 0;
    }

    jitter4testing(false);
  }

#if MDBX_LOCKING == MDBX_LOCKING_SYSV
  env->me_sysv_ipc.semid = -1;
#endif /* MDBX_LOCKING */

  if (current_pid != env->pid) {
    eASSERT0(env, !inprocess_neighbor);
    NOTICE("drown env %p after-fork pid %zd -> %zd", __Wpedantic_format_voidptr(env), (size_t)env->pid,
           (size_t)current_pid);
    inprocess_neighbor = nullptr;
  }

  /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored
   * after file was closed.
   *
   * 2) File locks would be released (by kernel) while the file-descriptors will
   * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel,
   * locks should be released here explicitly with properly order. */

  /* close dxb and restore lock */
  if (env->dsync_fd != INVALID_HANDLE_VALUE) {
    if (unlikely(close(env->dsync_fd) != 0) && rc == MDBX_SUCCESS)
      rc = errno;
    env->dsync_fd = INVALID_HANDLE_VALUE;
  }
  if (env->lazy_fd != INVALID_HANDLE_VALUE) {
    if (unlikely(close(env->lazy_fd) != 0) && rc == MDBX_SUCCESS)
      rc = errno;
    env->lazy_fd = INVALID_HANDLE_VALUE;
    if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
      /* restore file-lock */
      rc = lck_op(inprocess_neighbor->lazy_fd, F_SETLKW, (inprocess_neighbor->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
                  (inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? 0 : inprocess_neighbor->pid,
                  (inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1);
    }
  }

  /* close clk and restore locks */
  if (env->lck_mmap.fd != INVALID_HANDLE_VALUE) {
    if (unlikely(close(env->lck_mmap.fd) != 0) && rc == MDBX_SUCCESS)
      rc = errno;
    env->lck_mmap.fd = INVALID_HANDLE_VALUE;
    if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
      /* restore file-locks */
      rc = lck_op(inprocess_neighbor->lck_mmap.fd, F_SETLKW, F_RDLCK, 0, 1);
      if (rc == MDBX_SUCCESS && inprocess_neighbor->registered_reader_pid)
        rc = lck_rpid_set(inprocess_neighbor);
    }
  }

  if (inprocess_neighbor && rc != MDBX_SUCCESS)
    inprocess_neighbor->flags |= ENV_FATAL_ERROR;
  return rc;
}

/*---------------------------------------------------------------------------*/

__cold int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) {
#if MDBX_LOCKING == MDBX_LOCKING_SYSV
  int semid = -1;
  /* don't initialize semaphores twice */
  (void)inprocess_neighbor;
  if (global_uniqueness_flag == MDBX_RESULT_TRUE) {
    struct stat st;
    if (fstat(env->lazy_fd, &st))
      return errno;
  sysv_retry_create:
    semid = semget(env->me_sysv_ipc.key, 2, IPC_CREAT | IPC_EXCL | (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)));
    if (unlikely(semid == -1)) {
      int err = errno;
      if (err != EEXIST)
        return err;

      /* remove and re-create semaphore set */
      semid = semget(env->me_sysv_ipc.key, 2, 0);
      if (semid == -1) {
        err = errno;
        if (err != ENOENT)
          return err;
        goto sysv_retry_create;
      }
      if (semctl(semid, 2, IPC_RMID)) {
        err = errno;
        if (err != EIDRM)
          return err;
      }
      goto sysv_retry_create;
    }

    unsigned short val_array[2] = {1, 1};
    if (semctl(semid, 2, SETALL, val_array))
      return errno;
  } else {
    semid = semget(env->me_sysv_ipc.key, 2, 0);
    if (semid == -1)
      return errno;

    /* check read & write access */
    struct semid_ds data[2];
    if (semctl(semid, 2, IPC_STAT, data) || semctl(semid, 2, IPC_SET, data))
      return errno;
  }

  env->me_sysv_ipc.semid = semid;
  return MDBX_SUCCESS;

#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX
  (void)inprocess_neighbor;
  if (global_uniqueness_flag != MDBX_RESULT_TRUE)
    return MDBX_SUCCESS;
#error "FIXME: Not implemented"
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988

  /* don't initialize semaphores twice */
  (void)inprocess_neighbor;
  if (global_uniqueness_flag == MDBX_RESULT_TRUE) {
    if (sem_init(&env->lck_mmap.lck->rdt_lock, true, 1))
      return errno;
    if (sem_init(&env->lck_mmap.lck->wrt_lock, true, 1))
      return errno;
  }
  return MDBX_SUCCESS;

#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
  if (inprocess_neighbor)
    return MDBX_SUCCESS /* don't need any initialization for mutexes
      if LCK already opened/used inside current process */
        ;

  /* FIXME: Unfortunately, there is no other reliable way but to long testing
   * on each platform. On the other hand, behavior like FreeBSD is incorrect
   * and we can expect it to be rare. Moreover, even on FreeBSD without
   * additional in-process initialization, the probability of an problem
   * occurring is vanishingly small, and the symptom is a return of EINVAL
   * while locking a mutex. In other words, in the worst case, the problem
   * results in an EINVAL error at the start of the transaction, but NOT data
   * loss, nor database corruption, nor other fatal troubles. Thus, the code
   * below I am inclined to think the workaround for erroneous platforms (like
   * FreeBSD), rather than a defect of libmdbx. */
#if defined(__FreeBSD__)
  /* seems that shared mutexes on FreeBSD required in-process initialization */
  (void)global_uniqueness_flag;
#else
  /* shared mutexes on many other platforms (including Darwin and Linux's
   * futexes) doesn't need any addition in-process initialization */
  if (global_uniqueness_flag != MDBX_RESULT_TRUE)
    return MDBX_SUCCESS;
#endif

  pthread_mutexattr_t ma;
  int rc = pthread_mutexattr_init(&ma);
  if (rc)
    return rc;

  rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
  if (rc)
    goto bailout;

#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust)
  rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutexattr_setrobust_np)
  rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#elif _POSIX_THREAD_PROCESS_SHARED < 200809L
  rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#else
  rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#endif
  if (rc)
    goto bailout;
#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */

#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && !defined(MDBX_SAFE4QEMU)
  rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
  if (rc == ENOTSUP)
    rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
  if (rc && rc != ENOTSUP)
    goto bailout;
#endif /* PTHREAD_PRIO_INHERIT */

  rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
  if (rc && rc != ENOTSUP)
    goto bailout;

  rc = pthread_mutex_init(&env->lck_mmap.lck->rdt_lock, &ma);
  if (rc)
    goto bailout;
  rc = pthread_mutex_init(&env->lck_mmap.lck->wrt_lock, &ma);

bailout:
  pthread_mutexattr_destroy(&ma);
  return rc;
#else
#error "FIXME"
#endif /* MDBX_LOCKING > 0 */
}

__cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc, const int err) {
  int rc = err;
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV

#ifndef EOWNERDEAD
#define EOWNERDEAD MDBX_RESULT_TRUE
#endif /* EOWNERDEAD */

  if (err == EOWNERDEAD) {
    /* We own the mutex. Clean up after dead previous owner. */
    const bool rlocked = ipc == &env->lck->rdt_lock;
    rc = MDBX_SUCCESS;
    if (!rlocked) {
      if (unlikely(env->txn)) {
        /* env is hosed if the dead thread was ours */
        env->flags |= ENV_FATAL_ERROR;
        env->txn = nullptr;
        rc = MDBX_PANIC;
      }
    }
    WARNING("%clock owner died, %s", (rlocked ? 'r' : 'w'), (rc ? "this process' env is hosed" : "recovering"));

    int check_rc = mvcc_cleanup_dead(env, rlocked, nullptr);
    check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;

#if MDBX_LOCKING == MDBX_LOCKING_SYSV
    rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
#else
#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutex_consistent)
    int mreco_rc = pthread_mutex_consistent(ipc);
#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutex_consistent_np)
    int mreco_rc = pthread_mutex_consistent_np(ipc);
#elif _POSIX_THREAD_PROCESS_SHARED < 200809L
    int mreco_rc = pthread_mutex_consistent_np(ipc);
#else
    int mreco_rc = pthread_mutex_consistent(ipc);
#endif
    check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;

    if (unlikely(mreco_rc))
      ERROR("lock recovery failed, %s", mdbx_strerror(mreco_rc));

    rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
    if (MDBX_IS_ERROR(rc))
      pthread_mutex_unlock(ipc);
#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */
    return rc;
  }
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001
  (void)ipc;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
  (void)ipc;
#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX
#ifdef _MSC_VER
#pragma message("warning: TODO")
#else
#warning "TODO"
#endif
  (void)ipc;
#else
#error "FIXME"
#endif /* MDBX_LOCKING */

  ERROR("mutex (un)lock failed, %s", mdbx_strerror(err));
  if (rc != EDEADLK)
    env->flags |= ENV_FATAL_ERROR;
  return rc;
}

#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC)
int osal_check_tid4bionic(void) {
  /* avoid 32-bit Bionic bug/hang with 32-pit TID */
  if (sizeof(pthread_mutex_t) < sizeof(pid_t) + sizeof(unsigned)) {
    pid_t tid = gettid();
    if (unlikely(tid > 0xffff)) {
      FATAL("Raise the ENOSYS(%d) error to avoid hang due "
            "the 32-bit Bionic/Android bug with tid/thread_id 0x%08x(%i) "
            "that don’t fit in 16 bits, see "
            "https://android.googlesource.com/platform/bionic/+/master/"
            "docs/32-bit-abi.md#is-too-small-for-large-pids",
            ENOSYS, tid, tid);
      return ENOSYS;
    }
  }
  return 0;
}
#endif /* __ANDROID_API__ || ANDROID) || BIONIC */

static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, const bool dont_wait) {
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
  int rc = osal_check_tid4bionic();
  if (likely(rc == 0))
    rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc);
  rc = (rc == EBUSY && dont_wait) ? MDBX_BUSY : rc;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
  int rc = MDBX_SUCCESS;
  if (dont_wait) {
    if (sem_trywait(ipc)) {
      rc = errno;
      if (rc == EAGAIN)
        rc = MDBX_BUSY;
    }
  } else if (sem_wait(ipc))
    rc = errno;
#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
  struct sembuf op = {
      .sem_num = (ipc != &env->lck->wrt_lock), .sem_op = -1, .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO};
  int rc;
  if (semop(env->me_sysv_ipc.semid, &op, 1)) {
    rc = errno;
    if (dont_wait && rc == EAGAIN)
      rc = MDBX_BUSY;
  } else {
    rc = *ipc ? EOWNERDEAD : MDBX_SUCCESS;
    *ipc = env->pid;
  }
#else
#error "FIXME"
#endif /* MDBX_LOCKING */

  if (unlikely(rc != MDBX_SUCCESS && rc != MDBX_BUSY))
    rc = osal_ipclock_failed(env, ipc, rc);
  return rc;
}

static int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) {
  int err = MDBX_ENOSYS;
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
  err = pthread_mutex_unlock(ipc);
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
  err = sem_post(ipc) ? errno : MDBX_SUCCESS;
#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
  if (unlikely(*ipc != (pid_t)env->pid || env->me_sysv_ipc.key == -1))
    err = EPERM;
  else {
    *ipc = 0;
    struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock), .sem_op = 1, .sem_flg = SEM_UNDO};
    err = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS;
  }
#else
#error "FIXME"
#endif /* MDBX_LOCKING */
  int rc = err;
  if (unlikely(rc != MDBX_SUCCESS)) {
    const mdbx_pid_t current_pid = osal_getpid();
    if (current_pid == env->pid || LOG_ENABLED(MDBX_LOG_NOTICE))
      debug_log((current_pid == env->pid) ? MDBX_LOG_FATAL : (rc = MDBX_SUCCESS, MDBX_LOG_NOTICE), "ipc-unlock()",
                __LINE__, "failed: env %p, lck-%s %p, err %d\n", __Wpedantic_format_voidptr(env),
                (env->lck == env->lck_mmap.lck) ? "mmap" : "stub", __Wpedantic_format_voidptr(env->lck), err);
  }
  return rc;
}

int lck_rdt_lock(MDBX_env *env) {
  TRACE("%s", ">>");
  jitter4testing(true);
  int rc = osal_ipclock_lock(env, &env->lck->rdt_lock, false);
  TRACE("<< rc %d", rc);
  return rc;
}

void lck_rdt_unlock(MDBX_env *env) {
  TRACE("%s", ">>");
  int err = osal_ipclock_unlock(env, &env->lck->rdt_lock);
  TRACE("<< err %d", err);
  if (unlikely(err != MDBX_SUCCESS))
    panic_fmt(env, "%s-unlock error %d", "rdt", err);
  jitter4testing(true);
}

int lck_txn_lock(MDBX_env *env, bool dont_wait) {
  TRACE("%swait %s", dont_wait ? "dont-" : "", ">>");
  jitter4testing(true);
  const int err = osal_ipclock_lock(env, &env->lck->wrt_lock, dont_wait);
  int rc = err;
  if (likely(env->basal_txn && !MDBX_IS_ERROR(err))) {
    eASSERT0(env, !env->basal_txn->owner || err == /* если другой поток в этом-же процессе завершился
                                                     не освободив блокировку */
                                                MDBX_RESULT_TRUE);
    env->basal_txn->owner = osal_thread_self();
    rc = MDBX_SUCCESS;
  }
  TRACE("<< err %d, rc %d", err, rc);
  return rc;
}

void lck_txn_unlock(MDBX_env *env) {
  TRACE("%s", ">>");
  if (env->basal_txn) {
    eASSERT1(env, env->basal_txn->owner == osal_thread_self());
    env->basal_txn->owner = 0;
  }
  int err = osal_ipclock_unlock(env, &env->lck->wrt_lock);
  TRACE("<< err %d", err);
  if (unlikely(err != MDBX_SUCCESS))
    panic_fmt(env, "%s-unlock error %d", "wrt", err);
  jitter4testing(true);
}

#endif /* !Windows LCK-implementation */

#if defined(_WIN32) || defined(_WIN64)

/* PREAMBLE FOR WINDOWS:
 *
 * We are not concerned for performance here.
 * If you are running Windows a performance could NOT be the goal.
 * Otherwise please use Linux. */

#define LCK_SHARED 0
#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK
#define LCK_WAITFOR 0
#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY

static int flock_ex(HANDLE fd, HANDLE event, unsigned flags, size_t offset, size_t bytes, unsigned timeout_ms) {
  TRACE("lock>>: fd %p, timeout %u ms, event %p, flags 0x%x offset %zu, bytes %zu >>", fd, timeout_ms, event, flags,
        offset, bytes);
  ASSERT(timeout_ms == 0 || (event && event != INVALID_HANDLE_VALUE));
  ASSERT(timeout_ms == 0 || (flags & LCK_DONTWAIT) == 0);
  OVERLAPPED ov;
  ov.Internal = 0;
  ov.InternalHigh = 0;
  ov.hEvent = event;
  ov.Offset = (DWORD)offset;
  ov.OffsetHigh = HIGH_DWORD(offset);

  if (LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov)) {
    TRACE("lock<<: fd %p, timeout %u ms, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, timeout_ms, event,
          flags, offset, bytes, "done");
    return MDBX_SUCCESS;
  }

  DWORD rc = GetLastError();
  if (rc == ERROR_IO_PENDING) {
    if (timeout_ms) {
      rc = osal_waitstatus2errcode(WaitForSingleObject(event, timeout_ms));
      if (rc != MDBX_SUCCESS) {
        if (rc == ERROR_TIMEOUT)
          rc = ERROR_LOCK_VIOLATION;
        goto bailout;
      }
    }
    if (GetOverlappedResult(fd, &ov, &rc, true)) {
      TRACE("lock<<: fd %p, timeout %u ms, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, timeout_ms, event,
            flags, offset, bytes, "overlapped-done");
      return MDBX_SUCCESS;
    }
  bailout:
    CancelIo(fd);
  }

  TRACE("lock<<: fd %p, timeout %u ms, event %p, flags 0x%x offset %zu, bytes %zu << err %d", fd, timeout_ms, event,
        flags, offset, bytes, (int)rc);
  return (int)rc;
}

static int flock_lck(const MDBX_env *env, unsigned flags, size_t offset, size_t bytes, unsigned timeout_ms) {
  return flock_ex(env->lck_mmap.fd, env->lck_lock_event, flags, offset, bytes, timeout_ms);
}

static int flock_dxb(const MDBX_env *env, unsigned flags, size_t offset, size_t bytes) {
  const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
  return flock_ex(fd4data, env->dxb_lock_event, flags, offset, bytes, 0);
}

static int funlock(mdbx_filehandle_t fd, size_t offset, size_t bytes) {
  TRACE("unlock: fd %p, offset %zu, bytes %zu", fd, offset, bytes);
  return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, HIGH_DWORD(bytes)) ? MDBX_SUCCESS
                                                                                            : (int)GetLastError();
}

/*----------------------------------------------------------------------------*/
/* global `write` lock for write-txt processing,
 * exclusive locking both meta-pages) */

#ifdef _WIN64
#define DXB_MAXLEN UINT64_C(0x7fffFFFFfff00000)
#else
#define DXB_MAXLEN UINT32_C(0x7ff00000)
#endif
#define DXB_BODY (env->ps * (size_t)NUM_METAS), DXB_MAXLEN
#define DXB_WHOLE 0, DXB_MAXLEN

int lck_txn_lock(MDBX_env *env, bool dontwait) {
  if (dontwait) {
    if (!TryEnterCriticalSection(&env->dxb_event_cs))
      return MDBX_BUSY;
  } else {
    __try {
      EnterCriticalSection(&env->dxb_event_cs);
    } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
                    ? EXCEPTION_EXECUTE_HANDLER
                    : EXCEPTION_CONTINUE_SEARCH) {
      return MDBX_EDEADLK;
    }
  }

  eASSERT0(env, !env->basal_txn || !env->basal_txn->owner);
  if (env->flags & MDBX_EXCLUSIVE)
    goto done;

  int rc = flock_dxb(env, dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) : (LCK_EXCLUSIVE | LCK_WAITFOR), DXB_BODY);

  if (rc == MDBX_SUCCESS) {
  done:
    if (env->basal_txn)
      env->basal_txn->owner = osal_thread_self();
    /* Zap: Failing to release lock 'env->dxb_event_cs'
     *      in function 'mdbx_txn_lock' */
    MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(26115);
    return MDBX_SUCCESS;
  }

  LeaveCriticalSection(&env->dxb_event_cs);
  return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY;
}

void lck_txn_unlock(MDBX_env *env) {
  eASSERT1(env, !env->basal_txn || env->basal_txn->owner == osal_thread_self());
  if ((env->flags & MDBX_EXCLUSIVE) == 0) {
    int err = funlock(env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd, DXB_BODY);
    if (err != MDBX_SUCCESS)
      panic_fmt(env, "lck/%s-unlock error %d", "txn", err);
  }
  if (env->basal_txn)
    env->basal_txn->owner = 0;
  LeaveCriticalSection(&env->dxb_event_cs);
}

/*----------------------------------------------------------------------------*/
/* global `read` lock for readers registration,
 * exclusive locking `rdt_length` (second) cacheline */

#define LCK_LO_OFFSET 0
#define LCK_LO_LEN offsetof(lck_t, rdt_length)
#define LCK_UP_OFFSET LCK_LO_LEN
#define LCK_UP_LEN (sizeof(lck_t) - LCK_UP_OFFSET)
#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN
#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN

int lck_rdt_lock(MDBX_env *env) {
  int rc;
  imports.srwl_AcquireShared(&env->remap_lock);

  __try {
    EnterCriticalSection(&env->lck_event_cs);
  } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
                  ? EXCEPTION_EXECUTE_HANDLER
                  : EXCEPTION_CONTINUE_SEARCH) {
    rc = MDBX_EDEADLK;
    goto bailout;
  }

  if (env->lck_mmap.fd == INVALID_HANDLE_VALUE)
    goto done; /* readonly database in readonly filesystem */

  /* transition from S-? (used) to S-E (locked),
   * e.g. exclusive lock upper-part */
  if (env->flags & MDBX_EXCLUSIVE)
    goto done;

  rc = flock_lck(env, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER, 0);
  if (rc == MDBX_SUCCESS) {
  done:
    MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(26115);
    return MDBX_SUCCESS;
  }

bailout:
  imports.srwl_ReleaseShared(&env->remap_lock);
  return rc;
}

void lck_rdt_unlock(MDBX_env *env) {
  if (env->lck_mmap.fd != INVALID_HANDLE_VALUE && (env->flags & MDBX_EXCLUSIVE) == 0) {
    /* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */
    int err = funlock(env->lck_mmap.fd, LCK_UPPER);
    if (err != MDBX_SUCCESS)
      panic_fmt(env, "lck/%s-unlock error %d", "rdt", err);
  }
  LeaveCriticalSection(&env->lck_event_cs);
  imports.srwl_ReleaseShared(&env->remap_lock);
}

int osal_lockfile(mdbx_filehandle_t fd, bool wait) {
  return flock_ex(fd, 0, wait ? LCK_EXCLUSIVE | LCK_WAITFOR : LCK_EXCLUSIVE | LCK_DONTWAIT, 0, DXB_MAXLEN, 0);
}

static int suspend_and_append(mdbx_handle_array_t **array, const DWORD ThreadId) {
  const unsigned limit = (*array)->limit;
  if ((*array)->count == limit) {
    mdbx_handle_array_t *const ptr = osal_realloc(
        (limit > ARRAY_LENGTH((*array)->handles)) ? *array : /* don't free initial array on the stack */ nullptr,
        sizeof(mdbx_handle_array_t) + sizeof(HANDLE) * (limit * (size_t)2 - ARRAY_LENGTH((*array)->handles)));
    if (!ptr)
      return MDBX_ENOMEM;
    if (limit == ARRAY_LENGTH((*array)->handles))
      *ptr = **array;
    *array = ptr;
    (*array)->limit = limit * 2;
  }

  HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, FALSE, ThreadId);
  if (hThread == nullptr)
    return (int)GetLastError();

  if (SuspendThread(hThread) == (DWORD)-1) {
    int err = (int)GetLastError();
    DWORD ExitCode;
    if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || !GetExitCodeThread(hThread, &ExitCode) ||
        ExitCode != STILL_ACTIVE)
      err = MDBX_SUCCESS;
    CloseHandle(hThread);
    return err;
  }

  (*array)->handles[(*array)->count++] = hThread;
  return MDBX_SUCCESS;
}

int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
  eASSERT0(env, (env->flags & MDBX_NOSTICKYTHREADS) == 0);
  const uintptr_t CurrentTid = GetCurrentThreadId();
  int rc;
  if (env->lck_mmap.lck) {
    /* Scan LCK for threads of the current process */
    const reader_slot_t *const begin = env->lck_mmap.lck->rdt;
    const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease);
    const uintptr_t WriteTxnOwner = env->basal_txn ? env->basal_txn->owner : 0;
    for (const reader_slot_t *reader = begin; reader < end; ++reader) {
      if (reader->pid.weak != env->pid || !reader->tid.weak || reader->tid.weak >= MDBX_TID_TXN_OUSTED) {
      skip_lck:
        continue;
      }
      if (reader->tid.weak == CurrentTid || reader->tid.weak == WriteTxnOwner)
        goto skip_lck;

      rc = suspend_and_append(array, (mdbx_tid_t)reader->tid.weak);
      if (rc != MDBX_SUCCESS) {
      bailout_lck:
        (void)osal_resume_threads_after_remap(*array);
        return rc;
      }
    }
    if (WriteTxnOwner && WriteTxnOwner != CurrentTid) {
      rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner);
      if (rc != MDBX_SUCCESS)
        goto bailout_lck;
    }
  } else {
    /* Without LCK (i.e. read-only mode).
     * Walk through a snapshot of all running threads */
    eASSERT0(env, env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY));
    const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
    if (hSnapshot == INVALID_HANDLE_VALUE)
      return (int)GetLastError();

    THREADENTRY32 entry;
    entry.dwSize = sizeof(THREADENTRY32);

    if (!Thread32First(hSnapshot, &entry)) {
      rc = (int)GetLastError();
    bailout_toolhelp:
      CloseHandle(hSnapshot);
      (void)osal_resume_threads_after_remap(*array);
      return rc;
    }

    do {
      if (entry.th32OwnerProcessID != env->pid || entry.th32ThreadID == CurrentTid)
        continue;

      rc = suspend_and_append(array, entry.th32ThreadID);
      if (rc != MDBX_SUCCESS)
        goto bailout_toolhelp;

    } while (Thread32Next(hSnapshot, &entry));

    rc = (int)GetLastError();
    if (rc != ERROR_NO_MORE_FILES)
      goto bailout_toolhelp;
    CloseHandle(hSnapshot);
  }

  return MDBX_SUCCESS;
}

int osal_resume_threads_after_remap(mdbx_handle_array_t *array) {
  int rc = MDBX_SUCCESS;
  for (unsigned i = 0; i < array->count; ++i) {
    const HANDLE hThread = array->handles[i];
    if (ResumeThread(hThread) == (DWORD)-1) {
      const int err = (int)GetLastError();
      DWORD ExitCode;
      if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && GetExitCodeThread(hThread, &ExitCode) &&
          ExitCode == STILL_ACTIVE)
        rc = err;
    }
    CloseHandle(hThread);
  }
  return rc;
}

/*----------------------------------------------------------------------------*/
/* global `initial` lock for lockfile initialization,
 * exclusive/shared locking first cacheline */

/* Briefly description of locking schema/algorithm:
 *  - Windows does not support upgrading or downgrading for file locking.
 *  - Therefore upgrading/downgrading is emulated by shared and exclusive
 *    locking of upper and lower halves.
 *  - In other words, we have FSM with possible 9 states,
 *    i.e. free/shared/exclusive x free/shared/exclusive == 9.
 *    Only 6 states of FSM are used, which 2 of ones are transitive.
 *
 * States:
 *  LO HI
 *   ?-?  = free, i.e. unlocked
 *   S-?  = used, i.e. shared lock
 *   E-?  = exclusive-read, i.e. operational exclusive
 *   ?-S
 *   ?-E  = middle (transitive state)
 *   S-S
 *   S-E  = locked (transitive state)
 *   E-S
 *   E-E  = exclusive-write, i.e. exclusive due (re)initialization
 *
 *  The lck_seize() moves the locking-FSM from the initial free/unlocked
 *  state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible,
 *  or to the "used" (and returns MDBX_RESULT_FALSE).
 *
 *  The lck_downgrade() moves the locking-FSM from "exclusive write"
 *  state to the "used" (i.e. shared) state.
 *
 *  The lck_upgrade() moves the locking-FSM from "used" (i.e. shared)
 *  state to the "exclusive write" state.
 */

static void lck_unlock(MDBX_env *env) {
  int err;

  if (env->lck_mmap.fd != INVALID_HANDLE_VALUE) {
    /* double `unlock` for robustly remove overlapped shared/exclusive locks */
    do
      err = funlock(env->lck_mmap.fd, LCK_LOWER);
    while (err == MDBX_SUCCESS);
    ASSERT(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
    SetLastError(ERROR_SUCCESS);

    do
      err = funlock(env->lck_mmap.fd, LCK_UPPER);
    while (err == MDBX_SUCCESS);
    ASSERT(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
    SetLastError(ERROR_SUCCESS);
  }

  const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
  if (fd4data != INVALID_HANDLE_VALUE) {
    /* explicitly unlock to avoid latency for other processes (windows kernel
     * releases such locks via deferred queues) */
    do
      err = funlock(fd4data, DXB_BODY);
    while (err == MDBX_SUCCESS);
    ASSERT(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
    SetLastError(ERROR_SUCCESS);

    do
      err = funlock(fd4data, DXB_WHOLE);
    while (err == MDBX_SUCCESS);
    ASSERT(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
    SetLastError(ERROR_SUCCESS);
  }
}

#define TIMEOUT_SHORT_MS 121
#define TIMEOUT_LONG_MS 900000 /* 15 min */

/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE)
 * or as 'used' (S-? and returns MDBX_RESULT_FALSE).
 * Otherwise returns an error. */
static int internal_seize_lck(MDBX_env *env) {
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);

  /* 1) now on ?-? (free), get ?-E (middle) */
  jitter4testing(false);
  int rc = flock_lck(env, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER, TIMEOUT_LONG_MS);
  if (rc != MDBX_SUCCESS) {
    /* 2) something went wrong, give up */;
    ERROR("%s, err %u", "?-?(free) >> ?-E(middle)", rc);
    return rc;
  }

  /* 3) now on ?-E (middle), try E-E (exclusive-write) */
  jitter4testing(false);
  rc = flock_lck(env, LCK_EXCLUSIVE, LCK_LOWER, TIMEOUT_SHORT_MS);
  if (rc == MDBX_SUCCESS)
    return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */;

  /* 5) still on ?-E (middle) */
  jitter4testing(false);
  if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
    /* 6) something went wrong, give up */
    rc = funlock(env->lck_mmap.fd, LCK_UPPER);
    if (rc != MDBX_SUCCESS)
      panic_fmt(env, "lck/%s-unlock error %u", "?-E(middle) >> ?-?(free)", rc);
    return rc;
  }

  /* 7) still on ?-E (middle), try S-E (locked) */
  jitter4testing(false);
  rc = flock_lck(env, LCK_SHARED, LCK_LOWER, TIMEOUT_LONG_MS);

  jitter4testing(false);
  if (rc != MDBX_SUCCESS)
    ERROR("%s, err %u", "?-E(middle) >> S-E(locked)", rc);

  /* 8) now on S-E (locked) or still on ?-E (middle),
   *    transition to S-? (used) or ?-? (free) */
  int err = funlock(env->lck_mmap.fd, LCK_UPPER);
  if (err != MDBX_SUCCESS)
    panic_fmt(env, "lck/%s-unlock error %u", "X-E(locked/middle) >> X-?(used/free)", err);

  /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
  return rc;
}

int lck_seize(MDBX_env *env) {
  if (env->flags & MDBX_EXCLUSIVE)
    return MDBX_RESULT_TRUE /* nope since files were must be opened
                               non-shareable */
        ;

  if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) {
    /* LY: without-lck mode (e.g. on read-only filesystem) */
    jitter4testing(false);
    int rc = flock_dxb(env, LCK_SHARED | LCK_DONTWAIT, DXB_WHOLE);
    if (rc != MDBX_SUCCESS)
      ERROR("%s, err %u", "without-lck", rc);
    return rc;
  }

  int rc = internal_seize_lck(env);
  jitter4testing(false);
  if (rc == MDBX_RESULT_TRUE && (env->flags & MDBX_RDONLY) == 0) {
    /* Check that another process don't operates in without-lck mode.
     * Doing such check by exclusive locking the body-part of db. Should be
     * noted:
     *  - we need an exclusive lock for do so;
     *  - we can't lock meta-pages, otherwise other process could get an error
     *    while opening db in valid (non-conflict) mode. */
    int err = flock_dxb(env, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_WHOLE);
    if (err != MDBX_SUCCESS) {
      ERROR("%s, err %u", "lock-against-without-lck", err);
      jitter4testing(false);
      lck_unlock(env);
      return err;
    }
    jitter4testing(false);
    err = funlock(env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd, DXB_WHOLE);
    if (err != MDBX_SUCCESS)
      panic_fmt(env, "dxb/%s-unlock error %u", "lock-against-without-lck", err);
  }

  return rc;
}

int lck_downgrade(MDBX_env *env) {
  /* Transite from exclusive-write state (E-E) to used (S-?) */
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);

  if (env->flags & MDBX_EXCLUSIVE)
    return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
        ;
  /* 1) now at E-E (exclusive-write), transition to ?_E (middle) */
  int rc = funlock(env->lck_mmap.fd, LCK_LOWER);
  if (rc != MDBX_SUCCESS)
    panic_fmt(env, "lck/%s-unlock error %u", "E-E(exclusive-write) >> ?-E(middle)", rc);

  /* 2) now at ?-E (middle), transition to S-E (locked) */
  rc = flock_lck(env, LCK_SHARED, LCK_LOWER, TIMEOUT_LONG_MS);
  if (rc != MDBX_SUCCESS) {
    /* 3) something went wrong, give up */;
    ERROR("%s, err %u", "?-E(middle) >> S-E(locked)", rc);
    return rc;
  }

  /* 4) got S-E (locked), continue transition to S-? (used) */
  rc = funlock(env->lck_mmap.fd, LCK_UPPER);
  if (rc != MDBX_SUCCESS)
    panic_fmt(env, "lck/%s-unlock error %u", "S-E(locked) >> S-?(used)", rc);

  return MDBX_SUCCESS /* 5) now at S-? (used), done */;
}

int lck_upgrade(MDBX_env *env, bool dont_wait) {
  /* Transite from used state (S-?) to exclusive-write (E-E) */
  ASSERT(env->lck_mmap.fd != INVALID_HANDLE_VALUE);

  if (env->flags & MDBX_EXCLUSIVE)
    return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
        ;

  /* 1) now on S-? (used), try S-E (locked) */
  jitter4testing(false);
  int rc = flock_lck(env, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_UPPER, 0);
  if (rc != MDBX_SUCCESS) {
    /* 2) something went wrong, give up */;
    VERBOSE("%s, err %u", "S-?(used) >> S-E(locked)", rc);
    return rc;
  }

  /* 3) now on S-E (locked), transition to ?-E (middle) */
  rc = funlock(env->lck_mmap.fd, LCK_LOWER);
  if (rc != MDBX_SUCCESS)
    panic_fmt(env, "lck/%s-unlock error %u", "S-E(locked) >> ?-E(middle)", rc);

  /* 4) now on ?-E (middle), try E-E (exclusive-write) */
  jitter4testing(false);
  rc = flock_lck(env, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_LOWER, 0);
  if (rc != MDBX_SUCCESS) {
    /* 5) something went wrong, give up */;
    VERBOSE("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc);
    return rc;
  }

  return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */;
}

int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) {
  (void)env;
  (void)inprocess_neighbor;
  (void)global_uniqueness_flag;
  if (imports.SetFileIoOverlappedRange && !(env->flags & MDBX_RDONLY)) {
    HANDLE token = INVALID_HANDLE_VALUE;
    TOKEN_PRIVILEGES privileges;
    privileges.PrivilegeCount = 1;
    privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
    if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token) ||
        !LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME, &privileges.Privileges[0].Luid) ||
        !AdjustTokenPrivileges(token, FALSE, &privileges, sizeof(privileges), nullptr, nullptr) ||
        GetLastError() != ERROR_SUCCESS)
      imports.SetFileIoOverlappedRange = nullptr;

    if (token != INVALID_HANDLE_VALUE)
      CloseHandle(token);
  }
  return MDBX_SUCCESS;
}

int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const mdbx_pid_t current_pid) {
  (void)current_pid;
  /* LY: should unmap before releasing the locks to avoid race condition and
   * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */
  if (env->dxb_mmap.base)
    osal_munmap(&env->dxb_mmap);
  if (env->lck_mmap.lck) {
    const bool synced = env->lck_mmap.lck->unsynced_pages.weak == 0;
    osal_munmap(&env->lck_mmap);
    if (synced && !inprocess_neighbor && env->lck_mmap.fd != INVALID_HANDLE_VALUE &&
        lck_upgrade(env, true) == MDBX_SUCCESS)
      /* this will fail if LCK is used/mmapped by other process(es) */
      osal_fsetsize(env->lck_mmap.fd, 0);
  }
  lck_unlock(env);
  return MDBX_SUCCESS;
}

/*----------------------------------------------------------------------------*/
/* reader checking (by pid) */

int lck_rpid_set(MDBX_env *env) {
  (void)env;
  return MDBX_SUCCESS;
}

int lck_rpid_clear(MDBX_env *env) {
  (void)env;
  return MDBX_SUCCESS;
}

/* Checks reader by pid.
 *
 * Returns:
 *   MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
 *   MDBX_RESULT_FALSE, if pid is dead (lock acquired)
 *   or otherwise the errcode. */
int lck_rpid_check(MDBX_env *env, uint32_t pid) {
  (void)env;
  HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid);
  int rc;
  if (likely(hProcess)) {
    rc = WaitForSingleObject(hProcess, 0);
    if (unlikely(rc == (int)WAIT_FAILED))
      rc = (int)GetLastError();
    CloseHandle(hProcess);
  } else {
    rc = (int)GetLastError();
  }

  switch (rc) {
  case ERROR_INVALID_PARAMETER:
    /* pid seems invalid */
    return MDBX_RESULT_FALSE;
  case WAIT_OBJECT_0:
    /* process just exited */
    return MDBX_RESULT_FALSE;
  case ERROR_ACCESS_DENIED:
    /* The ERROR_ACCESS_DENIED would be returned for CSRSS-processes, etc.
     * assume pid exists */
    return MDBX_RESULT_TRUE;
  case WAIT_TIMEOUT:
    /* pid running */
    return MDBX_RESULT_TRUE;
  default:
    /* failure */
    return rc;
  }
}

#endif /* Windows */

__cold static int lck_setup_locked(MDBX_env *env) {
  int err = rthc_register(env);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  int lck_seize_rc = lck_seize(env);
  if (unlikely(MDBX_IS_ERROR(lck_seize_rc)))
    return lck_seize_rc;

  if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) {
    env->lck = lckless_stub(env);
    env->max_readers = UINT_MAX;
    DEBUG("lck-setup:%s%s%s", " lck-less", (env->flags & MDBX_RDONLY) ? " readonly" : "",
          (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative");
    return lck_seize_rc;
  }

  DEBUG("lck-setup:%s%s%s", " with-lck", (env->flags & MDBX_RDONLY) ? " readonly" : "",
        (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative");

  MDBX_env *inprocess_neighbor = nullptr;
  err = rthc_uniq_check(&env->lck_mmap, &inprocess_neighbor);
  if (unlikely(MDBX_IS_ERROR(err)))
    return err;
  if (inprocess_neighbor) {
    if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 || (inprocess_neighbor->flags & MDBX_EXCLUSIVE) != 0)
      return MDBX_BUSY;
    if (lck_seize_rc == MDBX_RESULT_TRUE) {
      err = lck_downgrade(env);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      lck_seize_rc = MDBX_RESULT_FALSE;
    }
  }

  uint64_t size = 0;
  err = osal_filesize(env->lck_mmap.fd, &size);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  if (lck_seize_rc == MDBX_RESULT_TRUE) {
    size = ceil_powerof2(env->max_readers * sizeof(reader_slot_t) + sizeof(lck_t), globals.sys_pagesize);
    jitter4testing(false);
  } else {
    if (env->flags & MDBX_EXCLUSIVE)
      return MDBX_BUSY;
    if (size > INT_MAX || (size & (globals.sys_pagesize - 1)) != 0 || size < globals.sys_pagesize) {
      ERROR("lck-file has invalid size %" PRIu64 " bytes", size);
      return MDBX_PROBLEM;
    }
  }

  const size_t maxreaders = ((size_t)size - sizeof(lck_t)) / sizeof(reader_slot_t);
  if (maxreaders < 4) {
    ERROR("lck-size too small (up to %" PRIuPTR " readers)", maxreaders);
    return MDBX_PROBLEM;
  }
  env->max_readers = (maxreaders <= MDBX_READERS_LIMIT) ? (unsigned)maxreaders : (unsigned)MDBX_READERS_LIMIT;

  err = osal_mmap((env->flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, &env->lck_mmap, (size_t)size, (size_t)size,
                  lck_seize_rc ? MMAP_OPTION_SETLENGTH | MMAP_OPTION_SEMAPHORE : MMAP_OPTION_SEMAPHORE,
                  env->pathname.lck);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

#ifdef MADV_DODUMP
  err = madvise(env->lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
  if (unlikely(MDBX_IS_ERROR(err)))
    return err;
#endif /* MADV_DODUMP */

#ifdef MADV_WILLNEED
  err = madvise(env->lck_mmap.lck, size, MADV_WILLNEED) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
  if (unlikely(MDBX_IS_ERROR(err)))
    return err;
#elif defined(POSIX_MADV_WILLNEED)
  err = ignore_enosys(posix_madvise(env->lck_mmap.lck, size, POSIX_MADV_WILLNEED));
  if (unlikely(MDBX_IS_ERROR(err)))
    return err;
#endif /* MADV_WILLNEED */

  lck_t *lck = env->lck_mmap.lck;
  if (lck_seize_rc == MDBX_RESULT_TRUE) {
    /* If we succeed got exclusive lock, then nobody is using the lock region
     * and we should initialize it. */
    memset(lck, 0, (size_t)size);
    jitter4testing(false);
    lck->magic_and_version = MDBX_LOCK_MAGIC;
    lck->os_and_format = MDBX_LOCK_FORMAT;
    lck->pgops.wops.weak = 1;
    err = osal_msync(&env->lck_mmap, (size_t)size, MDBX_SYNC_DATA | MDBX_SYNC_SIZE);
    if (unlikely(err != MDBX_SUCCESS)) {
      ERROR("initial-%s for lck-file failed, err %d", "msync/fsync", err);
      eASSERT0(env, MDBX_IS_ERROR(err));
      return err;
    }
  } else {
    if (lck->magic_and_version != MDBX_LOCK_MAGIC) {
      const bool invalid = (lck->magic_and_version >> 8) != MDBX_MAGIC;
      ERROR("lock region has %s", invalid ? "invalid magic"
                                          : "incompatible version (only applications with nearly or the "
                                            "same versions of libmdbx can share the same database)");
      return invalid ? MDBX_INVALID : MDBX_VERSION_MISMATCH;
    }
    if (lck->os_and_format != MDBX_LOCK_FORMAT) {
      ERROR("lock region has os/format signature 0x%" PRIx32 ", expected 0x%" PRIx32, lck->os_and_format,
            MDBX_LOCK_FORMAT);
      return MDBX_VERSION_MISMATCH;
    }
  }

  err = lck_init(env, inprocess_neighbor, lck_seize_rc);
  if (unlikely(err != MDBX_SUCCESS)) {
    eASSERT0(env, MDBX_IS_ERROR(err));
    return err;
  }

  env->lck = lck;
  eASSERT0(env, !MDBX_IS_ERROR(lck_seize_rc));
  return lck_seize_rc;
}

__cold int lck_setup(MDBX_env *env, mdbx_mode_t mode) {
  eASSERT0(env, env->lazy_fd != INVALID_HANDLE_VALUE);
  eASSERT0(env, env->lck_mmap.fd == INVALID_HANDLE_VALUE);

  int err = osal_openfile(MDBX_OPEN_LCK, env, env->pathname.lck, &env->lck_mmap.fd, mode);
  if (err != MDBX_SUCCESS) {
    switch (err) {
    case MDBX_EACCESS:
    case MDBX_EPERM:
      if (F_ISSET(env->flags, MDBX_RDONLY | MDBX_EXCLUSIVE))
        break;
      __fallthrough /* fall through */;
    case MDBX_ENOFILE:
    case MDBX_EROFS:
      if (env->flags & MDBX_RDONLY) {
        /* ENSURE the file system is read-only */
        int err_rofs = osal_check_fs_rdonly(env->lazy_fd, env->pathname.lck, err);
        if (err_rofs == MDBX_SUCCESS ||
            /* ignore ERROR_NOT_SUPPORTED for exclusive mode */
            (err_rofs == MDBX_ENOSYS && (env->flags & MDBX_EXCLUSIVE)))
          break;
        if (err_rofs != MDBX_ENOSYS)
          err = err_rofs;
      }
      __fallthrough /* fall through */;
    default:
      ERROR("unable to open lck-file %" MDBX_PRIsPATH ", env-flags 0x%X, err %d", env->pathname.lck, env->flags, err);
      return err;
    }

    /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
    env->lck_mmap.fd = INVALID_HANDLE_VALUE;
    NOTICE("continue %" MDBX_PRIsPATH " within without-lck mode, env-flags 0x%X, lck-error %d", env->pathname.dxb,
           env->flags, err);
  }

  rthc_lock();
  err = lck_setup_locked(env);
  rthc_unlock();
  return err;
}

void env_clear_incore_cache(const MDBX_env *const env) {
  memset(env->lck->mincore_cache.begin, -1, sizeof(env->lck->mincore_cache.begin));
}

/*------------------------------------------------------------------------------
 logging */

__cold void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args) {
  ENSURE(osal_fastmutex_acquire(&globals.debug_lock) == 0);
  if (globals.logger.ptr) {
    if (globals.logger_buffer == nullptr)
      globals.logger.fmt(level, function, line, fmt, args);
    else {
      const int len = vsnprintf(globals.logger_buffer, globals.logger_buffer_size, fmt, args);
      if (len > 0)
        globals.logger.nofmt(level, function, line, globals.logger_buffer, len);
    }
  } else {
#if defined(_WIN32) || defined(_WIN64)
    if (IsDebuggerPresent()) {
      int prefix_len = 0;
      char *prefix = nullptr;
      if (function && line > 0)
        prefix_len = osal_asprintf(&prefix, "%s:%d ", function, line);
      else if (function)
        prefix_len = osal_asprintf(&prefix, "%s: ", function);
      else if (line > 0)
        prefix_len = osal_asprintf(&prefix, "%d: ", line);
      if (prefix_len > 0 && prefix) {
        OutputDebugStringA(prefix);
        osal_free(prefix);
      }
      char *msg = nullptr;
      int msg_len = osal_vasprintf(&msg, fmt, args);
      if (msg_len > 0 && msg) {
        OutputDebugStringA(msg);
        osal_free(msg);
      }
    }
#else
    if (function && line > 0)
      fprintf(stderr, "%s:%d ", function, line);
    else if (function)
      fprintf(stderr, "%s: ", function);
    else if (line > 0)
      fprintf(stderr, "%d: ", line);
    vfprintf(stderr, fmt, args);
    fflush(stderr);
#endif
  }
  ENSURE(osal_fastmutex_release(&globals.debug_lock) == 0);
}

__cold void debug_log(int level, const char *function, int line, const char *fmt, ...) {
  va_list args;
  va_start(args, fmt);
  debug_log_va(level, function, line, fmt, args);
  va_end(args);
}

__cold void log_error(const int err, const char *func, unsigned line) {
  ASSERT(err != MDBX_SUCCESS);
  if (unlikely(globals.loglevel >= MDBX_LOG_DEBUG)) {
    const bool is_error = err != MDBX_RESULT_TRUE && err != MDBX_NOTFOUND;
    char buf[256];
    debug_log(is_error ? MDBX_LOG_ERROR : MDBX_LOG_VERBOSE, func, line, "%s %d (%s)\n",
              is_error ? "error" : "condition", err, mdbx_strerror_r(err, buf, sizeof(buf)));
  }
}

/* Dump a val in ascii or hexadecimal. */
__cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf, const size_t bufsize) {
  if (!val)
    return "<null>";
  if (!val->iov_len)
    return "<empty>";
  if (!buf || bufsize < 4)
    return nullptr;

  if (!val->iov_base) {
    int len = snprintf(buf, bufsize, "<nullptr.%zu>", val->iov_len);
    ASSERT(len > 0 && (size_t)len < bufsize);
    (void)len;
    return buf;
  }

  bool is_ascii = true;
  const uint8_t *const data = val->iov_base;
  for (size_t i = 0; i < val->iov_len; i++)
    if (data[i] < ' ' || data[i] > '~') {
      is_ascii = false;
      break;
    }

  if (is_ascii) {
    int len = snprintf(buf, bufsize, "%.*s", (val->iov_len > INT_MAX) ? INT_MAX : (int)val->iov_len, data);
    ASSERT(len > 0 && (size_t)len < bufsize);
    (void)len;
  } else {
    char *const detent = buf + bufsize - 2;
    char *ptr = buf;
    *ptr++ = '<';
    for (size_t i = 0; i < val->iov_len && ptr < detent; i++) {
      const char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
      *ptr++ = hex[data[i] >> 4];
      *ptr++ = hex[data[i] & 15];
    }
    if (ptr < detent)
      *ptr++ = '>';
    *ptr = '\0';
  }
  return buf;
}

__cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, union logger_union logger, char *buffer,
                              size_t buffer_size) {
  ENSURE(osal_fastmutex_acquire(&globals.debug_lock) == 0);

  const int rc = globals.runtime_flags | (globals.loglevel << 16);
  if (level != MDBX_LOG_DONTCHANGE) {
    level = clamp_unsigned(level, MDBX_LOG_FATAL, (MDBX_DEBUG > 0) ? MDBX_LOG_EXTRA : MDBX_LOG_NOTICE);
    globals.loglevel = (uint8_t)level;
  }

  if (flags != MDBX_DBG_DONTCHANGE) {
    flags &= MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE
#if MDBX_DEBUG > 0
             | MDBX_DBG_JITTER
#endif
#if MDBX_CHECKING > 1
             | MDBX_DBG_ASSERT
#endif
#if MDBX_CHECKING > 2
             | MDBX_DBG_AUDIT
#endif
        ;
    globals.runtime_flags = (uint8_t)flags;
  }

  ASSERT(MDBX_LOGGER_DONTCHANGE == ((MDBX_debug_func)(intptr_t)-1));
  if (logger.ptr != (void *)((intptr_t)-1)) {
    globals.logger.ptr = logger.ptr;
    globals.logger_buffer = buffer;
    globals.logger_buffer_size = buffer_size;
  }

  ENSURE(osal_fastmutex_release(&globals.debug_lock) == 0);
  return rc;
}

__cold int mdbx_setup_debug_nofmt(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func_nofmt logger,
                                  char *buffer, size_t buffer_size) {
  union logger_union thunk;
  thunk.nofmt = (logger && buffer && buffer_size) ? logger : MDBX_LOGGER_NOFMT_DONTCHANGE;
  return setup_debug(level, flags, thunk, buffer, buffer_size);
}

__cold int mdbx_setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func logger) {
  union logger_union thunk;
  thunk.fmt = logger;
  return setup_debug(level, flags, thunk, nullptr, 0);
}

/*------------------------------------------------------------------------------
 debug stuff */

__cold const char *pagetype_caption(const uint8_t type, char buf4unknown[16]) {
  switch (type) {
  case P_BRANCH:
    return "branch";
  case P_LEAF:
    return "leaf";
  case P_LEAF | P_SUBP:
    return "subleaf";
  case P_LEAF | P_DUPFIX:
    return "dupfix-leaf";
  case P_LEAF | P_DUPFIX | P_SUBP:
    return "dupfix-subleaf";
  case P_LEAF | P_DUPFIX | P_SUBP | P_LEGACY_DIRTY:
    return "dupfix-subleaf.legacy-dirty";
  case P_LARGE:
    return "large";
  default:
    snprintf(buf4unknown, 16, "unknown_0x%x", type);
    return buf4unknown;
  }
}

__cold static const char *leafnode_type(node_t *n) {
  static const char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}};
  return (node_flags(n) & N_BIG) ? ": large page" : tp[!!(node_flags(n) & N_DUP)][!!(node_flags(n) & N_TREE)];
}

/* Display all the keys in the page. */
__cold void page_list(page_t *mp) {
  pgno_t pgno = mp->pgno;
  const char *type;
  node_t *node;
  size_t i, nkeys, nsize, total = 0;
  MDBX_val key;
  DKBUF;

  switch (page_type(mp)) {
  case P_BRANCH:
    type = "Branch page";
    break;
  case P_LEAF:
    type = "Leaf page";
    break;
  case P_LEAF | P_SUBP:
    type = "Leaf sub-page";
    break;
  case P_LEAF | P_DUPFIX:
    type = "Leaf2 page";
    break;
  case P_LEAF | P_DUPFIX | P_SUBP:
    type = "Leaf2 sub-page";
    break;
  case P_LARGE:
    VERBOSE("Overflow page %" PRIaPGNO " pages %u\n", pgno, mp->pages);
    return;
  case P_META:
    VERBOSE("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno, unaligned_peek_u64(4, page_meta(mp)->txnid_a));
    return;
  default:
    VERBOSE("Bad page %" PRIaPGNO " flags 0x%X\n", pgno, mp->flags);
    return;
  }

  nkeys = page_numkeys(mp);
  VERBOSE("%s %" PRIaPGNO " numkeys %zu\n", type, pgno, nkeys);

  for (i = 0; i < nkeys; i++) {
    if (is_dupfix_leaf(mp)) { /* DUPFIX pages have no entries[] or node headers */
      key = page_dupfix_key(mp, i, nsize = mp->dupfix_ksize);
      total += nsize;
      VERBOSE("key %zu: nsize %zu, %s\n", i, nsize, DKEY(&key));
      continue;
    }
    node = page_node(mp, i);
    key.iov_len = node_ks(node);
    key.iov_base = node->payload;
    nsize = NODESIZE + key.iov_len;
    if (is_branch(mp)) {
      VERBOSE("key %zu: page %" PRIaPGNO ", %s\n", i, node_pgno(node), DKEY(&key));
      total += nsize;
    } else {
      if (node_flags(node) & N_BIG)
        nsize += sizeof(pgno_t);
      else
        nsize += node_ds(node);
      total += nsize;
      nsize += sizeof(indx_t);
      VERBOSE("key %zu: nsize %zu, %s%s\n", i, nsize, DKEY(&key), leafnode_type(node));
    }
    total = EVEN_CEIL(total);
  }
  VERBOSE("Total: header %u + contents %zu + unused %zu\n", is_dupfix_leaf(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->lower,
          total, page_room(mp));
}

#if MDBX_CHECKING >= 0

__cold const char *object2class(const void *ptr) {
  if (!ptr)
    return "null";

  int32_t snap_signature = 0;
  if (!osal_safe_peek_uint32(ptr, &snap_signature))
    return "bad";

  switch (snap_signature) {
  case env_signature:
    return "env";
  case txn_signature:
    return "txn";
  case cur_signature_live:
    return "cursor.live";
  case cur_signature_ready4dispose:
    return "cursor.r4clo";
  case cur_signature_wait4eot:
    return "cursor.w4eot";
  }

  return "unknown";
}

MDBX_NORETURN static void fuckup(const char *msg, const char *func, unsigned line, const void *obj) {
  const char *obj_class = object2class(obj);
  MDBX_DTRACE5(panic, func, line, msg, obj_class, obj);
  const MDBX_panic_func panic_func = globals.panic_func;
  if (panic_func)
    panic_func(msg, func, line, obj, obj_class);
  debug_log(MDBX_LOG_FATAL, func, line, obj ? "MDBX-ASSERTION: %s (%s %p)\n" : "MDBX-ASSERTION: %s\n", msg, obj_class,
            (void *)obj);
  osal_panic(msg, func, line);
}

__cold __noinline void panic_at_obj(const struct MDBX_panic_point *const at, const void *obj) {
  fuckup(at->msg, at->function, at->line, obj);
}

__cold __noinline void panic_at(const struct MDBX_panic_point *const at) { panic_at_obj(at, nullptr); }

__cold __noinline void panic_at_fmt(const struct MDBX_panic_point *const at, const void *obj, ...) {
  va_list ap;
  va_start(ap, obj);
  char *message = nullptr;
  const int num = osal_vasprintf(&message, at->msg, ap);
  const char *const const_message = unlikely(num < 1 || !message) ? "<vasprintf() failed>" : message;
  fuckup(const_message, at->function, at->line, obj);
}

__cold void mdbx_assert_fail(const char *msg, const char *func, unsigned line) { fuckup(msg, func, line, nullptr); }

#endif /* MDBX_CHECKING >= 0 */

typedef struct meta_snap {
  uint64_t txnid;
  size_t is_steady;
} meta_snap_t;

static inline txnid_t fetch_txnid(const volatile mdbx_atomic_uint32_t *ptr) {
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
  return atomic_load64((const volatile mdbx_atomic_uint64_t *)ptr, mo_AcquireRelease);
#else
  const uint32_t l = atomic_load32(&ptr[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease);
  const uint32_t h = atomic_load32(&ptr[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease);
  return (uint64_t)h << 32 | l;
#endif
}

static inline meta_snap_t meta_snap(const volatile meta_t *meta) {
  txnid_t txnid = fetch_txnid(meta->txnid_a);
  jitter4testing(true);
  size_t is_steady = meta_is_steady(meta) && txnid >= MIN_TXNID;
  jitter4testing(true);
  if (unlikely(txnid != fetch_txnid(meta->txnid_b)))
    txnid = is_steady = 0;
  meta_snap_t r = {txnid, is_steady};
  return r;
}

txnid_t meta_txnid(const volatile meta_t *meta) { return meta_snap(meta).txnid; }

meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n) {
  eASSERT0(env, n < NUM_METAS);
  meta_ptr_t r;
  meta_snap_t snap = meta_snap(r.ptr_v = METAPAGE(env, n));
  r.txnid = snap.txnid;
  r.is_steady = snap.is_steady;
  return r;
}

static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0, bool s1, bool s2) {
  ASSERT(c01 < 3 && c02 < 3 && c12 < 3);
  /* ASSERT(s0 < 2 && s1 < 2 && s2 < 2); */
  const uint8_t recent =
      meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2);
  const uint8_t prefer_steady =
      meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2);

  uint8_t tail;
  if (recent == 0)
    tail = meta_cmp2steady(c12, s1, s2) ? 2 : 1;
  else if (recent == 1)
    tail = meta_cmp2steady(c02, s0, s2) ? 2 : 0;
  else
    tail = meta_cmp2steady(c01, s0, s1) ? 1 : 0;

  const bool valid = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2;
  const bool strict = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2);
  return tail | recent << 2 | prefer_steady << 4 | strict << 6 | valid << 7;
}

static inline void meta_troika_unpack(troika_t *troika, const uint8_t packed) {
  troika->recent = (packed >> 2) & 3;
  troika->prefer_steady = (packed >> 4) & 3;
  troika->tail_and_flags = packed & 0xC3;
#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */
  troika->unused_pad = 0;
#endif
}

static const uint8_t troika_fsm_map[2 * 2 * 2 * 3 * 3 * 3] = {
    232, 201, 216, 216, 232, 233, 232, 232, 168, 201, 216, 152, 168, 233, 232, 168, 233, 201, 216, 201, 233, 233,
    232, 233, 168, 201, 152, 216, 232, 169, 232, 168, 168, 193, 152, 152, 168, 169, 232, 168, 169, 193, 152, 194,
    233, 169, 232, 169, 232, 201, 216, 216, 232, 201, 232, 232, 168, 193, 216, 152, 168, 193, 232, 168, 193, 193,
    210, 194, 225, 193, 225, 193, 168, 137, 212, 214, 232, 233, 168, 168, 168, 137, 212, 150, 168, 233, 168, 168,
    169, 137, 216, 201, 233, 233, 168, 169, 168, 137, 148, 214, 232, 169, 168, 168, 40,  129, 148, 150, 168, 169,
    168, 40,  169, 129, 152, 194, 233, 169, 168, 169, 168, 137, 214, 214, 232, 201, 168, 168, 168, 129, 214, 150,
    168, 193, 168, 168, 129, 129, 210, 194, 225, 193, 161, 129, 212, 198, 212, 214, 228, 228, 212, 212, 148, 201,
    212, 150, 164, 233, 212, 148, 233, 201, 216, 201, 233, 233, 216, 233, 148, 198, 148, 214, 228, 164, 212, 148,
    148, 194, 148, 150, 164, 169, 212, 148, 169, 194, 152, 194, 233, 169, 216, 169, 214, 198, 214, 214, 228, 198,
    212, 214, 150, 194, 214, 150, 164, 193, 212, 150, 194, 194, 210, 194, 225, 193, 210, 194};

__cold bool troika_verify_fsm(void) {
  bool ok = true;
  for (size_t i = 0; i < 2 * 2 * 2 * 3 * 3 * 3; ++i) {
    const bool s0 = (i >> 0) & 1;
    const bool s1 = (i >> 1) & 1;
    const bool s2 = (i >> 2) & 1;
    const uint8_t c01 = (i / (8 * 1)) % 3;
    const uint8_t c02 = (i / (8 * 3)) % 3;
    const uint8_t c12 = (i / (8 * 9)) % 3;

    const uint8_t packed = meta_cmp2pack(c01, c02, c12, s0, s1, s2);
    troika_t troika;
    troika.fsm = (uint8_t)i;
    meta_troika_unpack(&troika, packed);

    const uint8_t tail = TROIKA_TAIL(&troika);
    const bool strict = TROIKA_STRICT_VALID(&troika);
    const bool valid = TROIKA_VALID(&troika);

    const uint8_t recent_chk =
        meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2);
    const uint8_t prefer_steady_chk =
        meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2);

    uint8_t tail_chk;
    if (recent_chk == 0)
      tail_chk = meta_cmp2steady(c12, s1, s2) ? 2 : 1;
    else if (recent_chk == 1)
      tail_chk = meta_cmp2steady(c02, s0, s2) ? 2 : 0;
    else
      tail_chk = meta_cmp2steady(c01, s0, s1) ? 1 : 0;

    const bool valid_chk = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2;
    const bool strict_chk = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2);
    ASSERT(troika.recent == recent_chk);
    ASSERT(troika.prefer_steady == prefer_steady_chk);
    ASSERT(tail == tail_chk);
    ASSERT(valid == valid_chk);
    ASSERT(strict == strict_chk);
    ASSERT(troika_fsm_map[troika.fsm] == packed);
    if (troika.recent != recent_chk || troika.prefer_steady != prefer_steady_chk || tail != tail_chk ||
        valid != valid_chk || strict != strict_chk || troika_fsm_map[troika.fsm] != packed) {
      ok = false;
    }
  }
  return ok;
}

__hot troika_t meta_tap(const MDBX_env *env) {
  meta_snap_t snap;
  troika_t troika;
  snap = meta_snap(METAPAGE(env, 0));
  troika.txnid[0] = snap.txnid;
  troika.fsm = (uint8_t)snap.is_steady << 0;
  snap = meta_snap(METAPAGE(env, 1));
  troika.txnid[1] = snap.txnid;
  troika.fsm += (uint8_t)snap.is_steady << 1;
  troika.fsm += meta_cmp2int(troika.txnid[0], troika.txnid[1], 8);
  snap = meta_snap(METAPAGE(env, 2));
  troika.txnid[2] = snap.txnid;
  troika.fsm += (uint8_t)snap.is_steady << 2;
  troika.fsm += meta_cmp2int(troika.txnid[0], troika.txnid[2], 8 * 3);
  troika.fsm += meta_cmp2int(troika.txnid[1], troika.txnid[2], 8 * 3 * 3);

  meta_troika_unpack(&troika, troika_fsm_map[troika.fsm]);
  return troika;
}

txnid_t recent_committed_txnid(const MDBX_env *env) {
  const txnid_t m0 = meta_txnid(METAPAGE(env, 0));
  const txnid_t m1 = meta_txnid(METAPAGE(env, 1));
  const txnid_t m2 = meta_txnid(METAPAGE(env, 2));
  return (m0 > m1) ? ((m0 > m2) ? m0 : m2) : ((m1 > m2) ? m1 : m2);
}

static inline bool meta_eq(const troika_t *troika, size_t a, size_t b) {
  ASSERT(a < NUM_METAS && b < NUM_METAS);
  return troika->txnid[a] == troika->txnid[b] && (((troika->fsm >> a) ^ (troika->fsm >> b)) & 1) == 0 &&
         troika->txnid[a];
}

unsigned meta_eq_mask(const troika_t *troika) {
  return meta_eq(troika, 0, 1) | meta_eq(troika, 1, 2) << 1 | meta_eq(troika, 2, 0) << 2;
}

__hot bool meta_should_retry(const MDBX_env *env, troika_t *troika) {
  const troika_t prev = *troika;
  *troika = meta_tap(env);
  return prev.fsm != troika->fsm || prev.txnid[0] != troika->txnid[0] || prev.txnid[1] != troika->txnid[1] ||
         prev.txnid[2] != troika->txnid[2];
}

const char *durable_caption(const meta_t *const meta) {
  if (meta_is_steady(meta))
    return (meta_sign_get(meta) == meta_sign_calculate(meta)) ? "Steady" : "Tainted";
  return "Weak";
}

__cold void meta_troika_dump(const MDBX_env *env, const troika_t *troika) {
  const meta_ptr_t recent = meta_recent(env, troika);
  const meta_ptr_t prefer_steady = meta_prefer_steady(env, troika);
  const meta_ptr_t tail = meta_tail(env, troika);
  NOTICE("troika: %" PRIaTXN ".%c:%" PRIaTXN ".%c:%" PRIaTXN ".%c, fsm=0x%02x, "
         "head=%d-%" PRIaTXN ".%c, "
         "base=%d-%" PRIaTXN ".%c, "
         "tail=%d-%" PRIaTXN ".%c, "
         "valid %c, strict %c",
         troika->txnid[0], (troika->fsm & 1) ? 's' : 'w', troika->txnid[1], (troika->fsm & 2) ? 's' : 'w',
         troika->txnid[2], (troika->fsm & 4) ? 's' : 'w', troika->fsm, troika->recent, recent.txnid,
         recent.is_steady ? 's' : 'w', troika->prefer_steady, prefer_steady.txnid, prefer_steady.is_steady ? 's' : 'w',
         troika->tail_and_flags % NUM_METAS, tail.txnid, tail.is_steady ? 's' : 'w', TROIKA_VALID(troika) ? 'Y' : 'N',
         TROIKA_STRICT_VALID(troika) ? 'Y' : 'N');
}

/*----------------------------------------------------------------------------*/

static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto, const pgno_t pgno) {
  meta_t *const meta = METAPAGE(env, pgno);
  const txnid_t txnid = constmeta_txnid(meta);
  if (!meta_is_steady(meta) || txnid > inclusive_upto)
    return MDBX_RESULT_FALSE;

  WARNING("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, txnid, pgno);
  const uint64_t wipe = DATASIGN_NONE;
  const void *ptr = &wipe;
  size_t bytes = sizeof(meta->sign), offset = ptr_dist(&meta->sign, env->dxb_mmap.base);
  if (env->flags & MDBX_WRITEMAP) {
    unaligned_poke_u64(4, meta->sign, wipe);
    osal_flush_incoherent_cpu_writeback();
    if (!MDBX_AVOID_MSYNC)
      return MDBX_RESULT_TRUE;
    ptr = payload2page(meta);
    offset = ptr_dist(ptr, env->dxb_mmap.base);
    bytes = env->ps;
  }

  if (MDBX_ENABLE_PGOP_STAT)
    env->lck->pgops.wops.weak += 1;
  int err = osal_pwrite(env->fd4meta, ptr, bytes, offset);
  return likely(err == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : err;
}

__cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) {
  int err = meta_unsteady(env, inclusive_upto, 0);
  if (likely(!MDBX_IS_ERROR(err)))
    err = meta_unsteady(env, inclusive_upto, 1);
  if (likely(!MDBX_IS_ERROR(err)))
    err = meta_unsteady(env, inclusive_upto, 2);

  if (err == MDBX_RESULT_TRUE) {
    err = MDBX_SUCCESS;
    if (!MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP))
      err = dxb_msync(env, NUM_METAS, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
    else if (env->fd4meta == env->lazy_fd)
      err = dxb_fsync(env, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
  }

  osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize);

  /* force oldest refresh */
  atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed);

  env->basal_txn->wr.troika = meta_tap(env);
  for (MDBX_txn *scan = env->basal_txn->nested; scan; scan = scan->nested)
    scan->wr.troika = env->basal_txn->wr.troika;
  return err;
}

int meta_sync(const MDBX_env *env, const meta_ptr_t head) {
  eASSERT0(env, atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid);
  /* Функция может вызываться (в том числе) при (env->flags & MDBX_NOMETASYNC) == 0 и env->fd4meta == env->dsync_fd,
   * например если предыдущая транзакция была выполненна с флагом MDBX_NOMETASYNC. */

  int rc = MDBX_RESULT_TRUE;
  if (env->flags & MDBX_WRITEMAP) {
    if (!MDBX_AVOID_MSYNC)
      rc = dxb_msync(env, NUM_METAS, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
    else {
      if (MDBX_ENABLE_PGOP_STAT)
        env->lck->pgops.wops.weak += 1;
      const page_t *page = payload2page(head.ptr_c);
      rc = osal_pwrite(env->fd4meta, page, env->ps, ptr_dist(page, env->dxb_mmap.base));
      if (likely(rc == MDBX_SUCCESS) && env->fd4meta == env->lazy_fd)
        rc = dxb_fsync(env, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
    }
  } else
    rc = dxb_fsync(env, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);

  if (likely(rc == MDBX_SUCCESS))
    env->lck->meta_sync_txnid.weak = (uint32_t)head.txnid;
  return rc;
}

__cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num, const bin128_t *guid) {
  ENSURE_OBJ(env, is_powerof2(env->ps));
  ENSURE_OBJ(env, env->ps >= MDBX_MIN_PAGESIZE);
  ENSURE_OBJ(env, env->ps <= MDBX_MAX_PAGESIZE);
  ENSURE_OBJ(env, env->geo_in_bytes.lower >= MIN_MAPSIZE);
  ENSURE_OBJ(env, env->geo_in_bytes.upper <= MAX_MAPSIZE);
  ENSURE_OBJ(env, env->geo_in_bytes.now >= env->geo_in_bytes.lower);
  ENSURE_OBJ(env, env->geo_in_bytes.now <= env->geo_in_bytes.upper);

  memset(model, 0, env->ps);
  model->pgno = (pgno_t)num;
  model->flags = P_META;
  meta_t *const model_meta = page_meta(model);
  unaligned_poke_u64(4, model_meta->magic_and_version, MDBX_DATA_MAGIC);

  model_meta->geometry.lower = bytes2pgno(env, env->geo_in_bytes.lower);
  model_meta->geometry.upper = bytes2pgno(env, env->geo_in_bytes.upper);
  model_meta->geometry.grow_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.grow));
  model_meta->geometry.shrink_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink));
  model_meta->geometry.now = bytes2pgno(env, env->geo_in_bytes.now);
  model_meta->geometry.first_unallocated = NUM_METAS;

  ENSURE_OBJ(env, model_meta->geometry.lower >= MIN_PAGENO);
  ENSURE_OBJ(env, model_meta->geometry.upper <= MAX_PAGENO + 1);
  ENSURE_OBJ(env, model_meta->geometry.now >= model_meta->geometry.lower);
  ENSURE_OBJ(env, model_meta->geometry.now <= model_meta->geometry.upper);
  ENSURE_OBJ(env, model_meta->geometry.first_unallocated >= MIN_PAGENO);
  ENSURE_OBJ(env, model_meta->geometry.first_unallocated <= model_meta->geometry.now);
  ENSURE_OBJ(env, model_meta->geometry.grow_pv == pages2pv(pv2pages(model_meta->geometry.grow_pv)));
  ENSURE_OBJ(env, model_meta->geometry.shrink_pv == pages2pv(pv2pages(model_meta->geometry.shrink_pv)));

  model_meta->pagesize = env->ps;
  model_meta->trees.gc.flags = MDBX_INTEGERKEY;
  model_meta->trees.gc.root = P_INVALID;
  model_meta->trees.main.root = P_INVALID;
  memcpy(&model_meta->dxbid, guid, sizeof(model_meta->dxbid));
  meta_set_txnid(env, model_meta, MIN_TXNID + num);
  unaligned_poke_u64(4, model_meta->sign, meta_sign_calculate(model_meta));
  eASSERT1(env, coherency_check_meta(env, model_meta, true));
  return ptr_disp(model, env->ps);
}

__cold meta_t *meta_init_triplet(const MDBX_env *env, void *buffer) {
  const bin128_t guid = osal_guid(env);
  page_t *page0 = (page_t *)buffer;
  page_t *page1 = meta_model(env, page0, 0, &guid);
  page_t *page2 = meta_model(env, page1, 1, &guid);
  meta_model(env, page2, 2, &guid);
  return page_meta(page2);
}

__cold int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape) {
  page_t *const page = env->page_auxbuf;
  meta_model(env, page, target, &((target == 0 && shape) ? shape : METAPAGE(env, 0))->dxbid);
  meta_t *const model = page_meta(page);
  meta_set_txnid(env, model, txnid);
  if (txnid)
    eASSERT1(env, coherency_check_meta(env, model, true));
  if (shape) {
    if (txnid && unlikely(!coherency_check_meta(env, shape, false))) {
      ERROR("bailout overriding meta-%zu since model failed "
            "FreeDB/MainDB %s-check for txnid #%" PRIaTXN,
            target, "pre", constmeta_txnid(shape));
      return MDBX_PROBLEM;
    }
    if (globals.runtime_flags & MDBX_DBG_DONT_UPGRADE)
      memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version));
    model->reserve16 = shape->reserve16;
    model->validator_id = shape->validator_id;
    model->extra_pagehdr = shape->extra_pagehdr;
    memcpy(&model->geometry, &shape->geometry, sizeof(model->geometry));
    memcpy(&model->trees, &shape->trees, sizeof(model->trees));
    memcpy(&model->canary, &shape->canary, sizeof(model->canary));
    memcpy(&model->pages_retired, &shape->pages_retired, sizeof(model->pages_retired));
    if (txnid) {
      if ((!model->trees.gc.mod_txnid && model->trees.gc.root != P_INVALID) ||
          (!model->trees.main.mod_txnid && model->trees.main.root != P_INVALID))
        memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version));
      if (unlikely(!coherency_check_meta(env, model, false))) {
        ERROR("bailout overriding meta-%zu since model failed "
              "FreeDB/MainDB %s-check for txnid #%" PRIaTXN,
              target, "post", txnid);
        return MDBX_PROBLEM;
      }
    }
  }

  if (target == 0 && (model->dxbid.x | model->dxbid.y) == 0) {
    const bin128_t guid = osal_guid(env);
    memcpy(&model->dxbid, &guid, sizeof(model->dxbid));
  }

  meta_sign_as_steady(model);
  int rc = meta_validate(env, model, page, (pgno_t)target, nullptr);
  if (unlikely(MDBX_IS_ERROR(rc)))
    return MDBX_PROBLEM;

  if (shape && memcmp(model, shape, sizeof(meta_t)) == 0) {
    NOTICE("skip overriding meta-%zu since no changes "
           "for txnid #%" PRIaTXN,
           target, txnid);
    return MDBX_SUCCESS;
  }

  if (env->flags & MDBX_WRITEMAP) {
    if (MDBX_ENABLE_PGOP_STAT)
      env->lck->pgops.msync.weak += 1;
    rc = dxb_msync(env, model->geometry.first_unallocated, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    /* meta_override() called only while current process have exclusive
     * lock of a DB file. So meta-page could be updated directly without
     * clearing consistency flag by mdbx_meta_update_begin() */
    memcpy(pgno2page(env, target), page, env->ps);
    osal_flush_incoherent_cpu_writeback();
    rc = dxb_msync(env, target + 1, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
  } else {
    if (MDBX_ENABLE_PGOP_STAT)
      env->lck->pgops.wops.weak += 1;
    rc = osal_pwrite(env->fd4meta, page, env->ps, pgno2bytes(env, target));
    if (rc == MDBX_SUCCESS && env->fd4meta == env->lazy_fd)
      rc = dxb_fsync(env, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
    osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize);
  }
  eASSERT0(env,
           (!env->txn && (env->flags & ENV_ACTIVE) == 0) ||
               (env->stuck_meta == (int)target && (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE));
  return rc;
}

__cold int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number,
                         unsigned *guess_pagesize) {
  const uint64_t magic_and_version = unaligned_peek_u64(4, &meta->magic_and_version);
  if (unlikely(magic_and_version != MDBX_DATA_MAGIC && magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT &&
               magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) {
    ERROR("meta[%u] has invalid magic/version %" PRIx64, meta_number, magic_and_version);
    return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID : MDBX_VERSION_MISMATCH;
  }

  if (unlikely(page->pgno != meta_number)) {
    ERROR("meta[%u] has invalid pageno %" PRIaPGNO, meta_number, page->pgno);
    return MDBX_INVALID;
  }

  if (unlikely(page->flags != P_META)) {
    ERROR("page #%u not a meta-page", meta_number);
    return MDBX_INVALID;
  }

  if (unlikely(!is_powerof2(meta->pagesize) || meta->pagesize < MDBX_MIN_PAGESIZE ||
               meta->pagesize > MDBX_MAX_PAGESIZE)) {
    WARNING("meta[%u] has invalid pagesize (%u), skip it", meta_number, meta->pagesize);
    return is_powerof2(meta->pagesize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID;
  }

  if (guess_pagesize && *guess_pagesize != meta->pagesize) {
    *guess_pagesize = meta->pagesize;
    VERBOSE("meta[%u] took pagesize %u", meta_number, meta->pagesize);
  }

  const txnid_t txnid = unaligned_peek_u64(4, &meta->txnid_a);
  if (unlikely(txnid != unaligned_peek_u64(4, &meta->txnid_b))) {
    WARNING("meta[%u] not completely updated, skip it", meta_number);
    return MDBX_RESULT_TRUE;
  }

  /* LY: check signature as a checksum */
  const uint64_t sign = meta_sign_get(meta);
  const uint64_t sign_stready = meta_sign_calculate(meta);
  if (SIGN_IS_STEADY(sign) && unlikely(sign != sign_stready)) {
    WARNING("meta[%u] has invalid steady-checksum (0x%" PRIx64 " != 0x%" PRIx64 "), skip it", meta_number, sign,
            sign_stready);
    return MDBX_RESULT_TRUE;
  }

  if (unlikely(meta->trees.gc.flags != MDBX_INTEGERKEY) &&
      ((meta->trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY || magic_and_version == MDBX_DATA_MAGIC)) {
    WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "GC/FreeDB", meta->trees.gc.flags);
    return MDBX_INCOMPATIBLE;
  }

  if (unlikely(!check_table_flags(meta->trees.main.flags))) {
    WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "MainDB", meta->trees.main.flags);
    return MDBX_INCOMPATIBLE;
  }

  DEBUG("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO
        "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s",
        page->pgno, meta->trees.main.root, meta->trees.gc.root, meta->geometry.lower, meta->geometry.first_unallocated,
        meta->geometry.now, meta->geometry.upper, pv2pages(meta->geometry.grow_pv), pv2pages(meta->geometry.shrink_pv),
        txnid, durable_caption(meta));

  if (unlikely(txnid < MIN_TXNID || txnid > MAX_TXNID)) {
    WARNING("meta[%u] has invalid txnid %" PRIaTXN ", skip it", meta_number, txnid);
    return MDBX_RESULT_TRUE;
  }

  if (unlikely(meta->geometry.lower < MIN_PAGENO || meta->geometry.lower > MAX_PAGENO + 1)) {
    WARNING("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.lower);
    return MDBX_INVALID;
  }

  if (unlikely(meta->geometry.upper < MIN_PAGENO || meta->geometry.upper > MAX_PAGENO + 1 ||
               meta->geometry.upper < meta->geometry.lower)) {
    WARNING("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.upper);
    return MDBX_INVALID;
  }

  if (unlikely(meta->geometry.first_unallocated < MIN_PAGENO || meta->geometry.first_unallocated - 1 > MAX_PAGENO)) {
    WARNING("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", meta_number, meta->geometry.first_unallocated);
    return MDBX_CORRUPTED;
  }

  const uint64_t allocated_bytes = meta->geometry.first_unallocated * (uint64_t)meta->pagesize;
  const uint64_t dxbsize_pages = env->dxb_mmap.filesize / (uint64_t)meta->pagesize;
  if (unlikely(allocated_bytes > env->dxb_mmap.filesize)) {
    /* Here could be a race with DB-shrinking performed by other process */
    int err = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if (unlikely(allocated_bytes > env->dxb_mmap.filesize)) {
      WARNING("meta[%u] allocated-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 "), skip it", meta_number,
              allocated_bytes, env->dxb_mmap.filesize);
      return MDBX_CORRUPTED;
    }
  }
  if (unlikely(meta->geometry.first_unallocated - 1 > MAX_PAGENO || allocated_bytes > MAX_MAPSIZE)) {
    WARNING("meta[%u] has too large allocated-space (%" PRIu64 "), skip it", meta_number, allocated_bytes);
    return MDBX_TOO_LARGE;
  }

  pgno_t geo_lower = meta->geometry.lower;
  uint64_t mapsize_min = geo_lower * (uint64_t)meta->pagesize;
  STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MDBX_MAX_PAGESIZE);
  STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
  STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MDBX_MIN_PAGESIZE % (4ul << 20) == 0);
  if (unlikely(mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE)) {
    if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE && mapsize_min <= MAX_MAPSIZE64) {
      eASSERT0(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && allocated_bytes <= MAX_MAPSIZE);
      WARNING("meta[%u] has too large min-mapsize (%" PRIu64 "), "
              "but size of allocated space still acceptable (%" PRIu64 ")",
              meta_number, mapsize_min, allocated_bytes);
      geo_lower = (pgno_t)((mapsize_min = MAX_MAPSIZE) / meta->pagesize);
      if (geo_lower > MAX_PAGENO + 1) {
        geo_lower = MAX_PAGENO + 1;
        mapsize_min = geo_lower * (uint64_t)meta->pagesize;
      }
      WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO
              ", will be corrected on next commit(s)",
              meta_number, "lower", geo_lower, meta->geometry.lower);
      meta->geometry.lower = geo_lower;
    } else {
      WARNING("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_min);
      return MDBX_VERSION_MISMATCH;
    }
  }

  /* It has already been verified above that the size of the allocated space does not exceed the file size. */
  pgno_t geo_upper = meta->geometry.upper;
  uint64_t mapsize_max = geo_upper * (uint64_t)meta->pagesize;
  STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
  if (unlikely(mapsize_max > MAX_MAPSIZE ||
               (MAX_PAGENO + 1) < ceil_powerof2((/* допустимо, так как уже проверили что mapsize_max <= MAX_MAPSIZE,
                                                  * а следовательно mapsize_max помещается в size_t */
                                                 size_t)mapsize_max,
                                                globals.sys_allocation_granularity) /
                                      meta->pagesize)) {
    if (mapsize_max > MAX_MAPSIZE64) {
      WARNING("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_max);
      return MDBX_VERSION_MISMATCH;
    }
    /* allow to open large DB from a 32-bit environment */
    eASSERT0(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && allocated_bytes <= MAX_MAPSIZE);
    WARNING("meta[%u] has too large max-mapsize (%" PRIu64 "), "
            "but size of allocated space still acceptable (%" PRIu64 ")",
            meta_number, mapsize_max, allocated_bytes);
    geo_upper = (pgno_t)((mapsize_max = MAX_MAPSIZE) / meta->pagesize);
    if (geo_upper > MAX_PAGENO + 1)
      geo_upper = MAX_PAGENO + 1;
  }

  if (geo_upper < meta->geometry.first_unallocated) {
    WARNING("meta[%u] has too less max-mapsize (%" PRIu64 "), "
            "but size of allocated space still acceptable (%" PRIu64 ")",
            meta_number, mapsize_max, allocated_bytes);
    geo_upper = meta->geometry.first_unallocated;
  }
  if (meta->geometry.upper != geo_upper) {
    WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO
            ", will be corrected on next commit(s)",
            meta_number, "upper", geo_upper, meta->geometry.upper);
    meta->geometry.upper = geo_upper;
    mapsize_max = geo_upper * (uint64_t)meta->pagesize;
  }

  /* LY: check and silently put geometry.now into [geo.lower...geo.upper].
   *
   * It has already been verified above that the size of the allocated space does not exceed the file size.
   *
   * Copy-with-compaction by old version of libmdbx could produce DB-file less than meta.geo.lower bound, in case actual
   * filling is low or no data at all. This is not a problem as there is no damage or loss of data. Therefore it is
   * better not to consider such situation as an error, but silently correct it. */
  pgno_t geo_now = meta->geometry.now;
  if (geo_now < geo_lower)
    geo_now = geo_lower;
  if (geo_now < dxbsize_pages)
    geo_now = dxbsize_pages;
  if (geo_now > geo_upper)
    geo_now = geo_upper;

  if (unlikely(/* paranoid */ meta->geometry.first_unallocated > geo_now)) {
    WARNING("meta[%u] next-pageno (%" PRIaPGNO ") is beyond end-pgno (%" PRIaPGNO "), skip it", meta_number,
            meta->geometry.first_unallocated, geo_now);
    return MDBX_CORRUPTED;
  }
  if (meta->geometry.now != geo_now) {
    WARNING("meta[%u] consider geo-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO
            ", will be corrected on next commit(s)",
            meta_number, "now", geo_now, meta->geometry.now);
    meta->geometry.now = geo_now;
  }

  /* GC */
  if (meta->trees.gc.root == P_INVALID) {
    if (unlikely(meta->trees.gc.branch_pages || meta->trees.gc.height || meta->trees.gc.items ||
                 meta->trees.gc.leaf_pages || meta->trees.gc.large_pages)) {
      WARNING("meta[%u] has false-empty %s, skip it", meta_number, "GC");
      return MDBX_CORRUPTED;
    }
  } else if (unlikely(meta->trees.gc.root >= meta->geometry.first_unallocated || meta->trees.gc.root < NUM_METAS)) {
    WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "GC", meta->trees.gc.root);
    return MDBX_CORRUPTED;
  }

  /* MainDB */
  if (meta->trees.main.root == P_INVALID) {
    if (unlikely(meta->trees.main.branch_pages || meta->trees.main.height || meta->trees.main.items ||
                 meta->trees.main.leaf_pages || meta->trees.main.large_pages)) {
      WARNING("meta[%u] has false-empty %s", meta_number, "MainDB");
      return MDBX_CORRUPTED;
    }
  } else if (unlikely(meta->trees.main.root >= meta->geometry.first_unallocated || meta->trees.main.root < NUM_METAS)) {
    WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "MainDB", meta->trees.main.root);
    return MDBX_CORRUPTED;
  }

  if (unlikely(meta->trees.gc.mod_txnid > txnid)) {
    WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.gc.mod_txnid, "GC");
    return MDBX_CORRUPTED;
  }

  if (unlikely(meta->trees.main.mod_txnid > txnid)) {
    WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.main.mod_txnid,
            "MainDB");
    return MDBX_CORRUPTED;
  }

  return MDBX_SUCCESS;
}

__cold int meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest) {
  *dest = *meta;
  return meta_validate(env, dest, payload2page(meta), bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)), nullptr);
}

bsr_t mvcc_bind_slot(MDBX_env *env) {
  eASSERT0(env, env->lck_mmap.lck);
  eASSERT0(env, env->lck->magic_and_version == MDBX_LOCK_MAGIC);
  eASSERT0(env, env->lck->os_and_format == MDBX_LOCK_FORMAT);

  bsr_t result = {lck_rdt_lock(env), nullptr};
  if (unlikely(MDBX_IS_ERROR(result.err)))
    return result;
  if (unlikely(env->flags & ENV_FATAL_ERROR)) {
    lck_rdt_unlock(env);
    result.err = MDBX_PANIC;
    return result;
  }
  if (unlikely(!env->dxb_mmap.base)) {
    lck_rdt_unlock(env);
    result.err = MDBX_EPERM;
    return result;
  }

  if (unlikely(env->registered_reader_pid != env->pid)) {
    result.err = lck_rpid_set(env);
    if (unlikely(result.err != MDBX_SUCCESS)) {
      lck_rdt_unlock(env);
      return result;
    }
    env->registered_reader_pid = env->pid;
  }

  result.err = MDBX_SUCCESS;
  size_t slot, nreaders;
  while (1) {
    nreaders = env->lck->rdt_length.weak;
    for (slot = 0; slot < nreaders; slot++)
      if (!atomic_load_pid(&env->lck->rdt[slot].pid, mo_AcquireRelease))
        break;

    if (likely(slot < env->max_readers))
      break;

    result.err = mvcc_cleanup_dead(env, true, nullptr);
    if (result.err != MDBX_RESULT_TRUE) {
      lck_rdt_unlock(env);
      result.err = (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err;
      return result;
    }
  }

  result.slot = &env->lck->rdt[slot];
  /* Claim the reader slot, carefully since other code
   * uses the reader table un-mutexed: First reset the
   * slot, next publish it in lck->rdt_length.  After
   * that, it is safe for mdbx_env_close() to touch it.
   * When it will be closed, we can finally claim it. */
  atomic_store32(&result.slot->pid, 0, mo_AcquireRelease);
  safe64_reset(&result.slot->txnid, true);
  if (slot == nreaders)
    env->lck->rdt_length.weak = (uint32_t)++nreaders;
  result.slot->tid.weak = (env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self();
  atomic_store32(&result.slot->pid, env->pid, mo_AcquireRelease);
  lck_rdt_unlock(env);

  if (likely(env->flags & ENV_TXKEY)) {
    eASSERT0(env, env->registered_reader_pid == env->pid);
    thread_rthc_set(env->me_txkey, result.slot);
  }
  return result;
}

#define NOTHING_CHANGED_SIGNATURE MDBX_TETRAD('N', 'o', 'n', 'e')

MDBX_MAYBE_UNUSED __hot orsi_ro_t mvcc_shapshot_oldest_ro(const MDBX_txn *const txn,
                                                          const bool need_thisprocess_oldest) {
  const uint32_t nothing_changed_signature = NOTHING_CHANGED_SIGNATURE;
  cASSERT0(txn, (txn->flags & txn_ro_flat) != 0);
  orsi_ro_t result;
  lck_t *const lck = txn->env->lck;
  result.nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
  result.thisprocess_oldest_txnid = result.oldest_txnid = result.recent_txnid = recent_committed_txnid(txn->env);

  if (!need_thisprocess_oldest &&
      nothing_changed_signature == atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) {
    result.oldest_txnid = atomic_load64(&lck->cached_oldest_txnid, mo_AcquireRelease);
    if (nothing_changed_signature == atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease))
      return result;
  }

  for (size_t i = 0; i < result.nreaders; ++i) {
    const mdbx_pid_t pid = atomic_load_pid(&lck->rdt[i].pid, mo_AcquireRelease);
    if (!pid)
      continue;

    jitter4testing(true);
    const txnid_t reader_txnid = safe64_read(&lck->rdt[i].txnid);
    result.oldest_txnid = (result.oldest_txnid > reader_txnid) ? reader_txnid : result.oldest_txnid;
    if (pid == txn->env->registered_reader_pid && result.thisprocess_oldest_txnid > reader_txnid)
      result.thisprocess_oldest_txnid = reader_txnid;
  }

  return result;
}

__hot orsi_rw_t mvcc_shapshot_oldest_rw(const MDBX_txn *const txn) {
  const uint32_t nothing_changed_signature = NOTHING_CHANGED_SIGNATURE;
  cASSERT0(txn, (txn->flags & txn_ro_flat) == 0);
  lck_t *const lck = txn->env->lck;
  uint64_t oldest_retired_pages = lck->cached_oldest_retired.weak;
  const txnid_t prev_oldest = atomic_load64(&lck->cached_oldest_txnid, mo_AcquireRelease);
  const meta_ptr_t steady = meta_prefer_steady(txn->env, &txn->wr.troika);
  orsi_rw_t result = {.steady_txnid = steady.txnid, .oldest_txnid = prev_oldest};
  cASSERT0(txn, steady.txnid <= txn->env->basal_txn->txnid);
  cASSERT0(txn, steady.txnid >= prev_oldest);

  while (nothing_changed_signature != atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) {
    lck->rdt_refresh_flag.weak = nothing_changed_signature;
    jitter4testing(false);
    const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
    result.oldest_txnid = result.steady_txnid;
    oldest_retired_pages = unaligned_peek_u64(4, steady.ptr_c->pages_retired);

    for (size_t i = 0; i < snap_nreaders; ++i) {
    retry:;
      const mdbx_pid_t pid = atomic_load_pid(&lck->rdt[i].pid, mo_AcquireRelease);
      if (!pid)
        continue;
      jitter4testing(true);

      const uint64_t reader_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed);
      const txnid_t reader_txnid = atomic_load64(&lck->rdt[i].txnid, mo_Relaxed);
      if (unlikely(reader_retired != atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_AcquireRelease) ||
                   reader_txnid != atomic_load64(&lck->rdt[i].txnid, mo_AcquireRelease))) {
        atomic_yield();
        goto retry;
      }

      if (unlikely(reader_txnid < prev_oldest)) {
        if (unlikely(nothing_changed_signature == atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) &&
            safe64_reset_compare(&lck->rdt[i].txnid, reader_txnid)) {
          NOTICE("kick stuck reader[%zu of %zu].pid_%zu %" PRIaTXN " < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN,
                 i, snap_nreaders, (size_t)pid, reader_txnid, prev_oldest, steady.txnid);
        }
        continue;
      }

      if (reader_txnid < result.oldest_txnid) {
        result.oldest_txnid = reader_txnid;
        oldest_retired_pages = reader_retired;
        if (MDBX_CHECKING < 1 && result.oldest_txnid == prev_oldest)
          break;
      }
    }
  }

  if (result.oldest_txnid != prev_oldest) {
    VERBOSE("update oldest %" PRIaTXN " -> %" PRIaTXN, prev_oldest, result.oldest_txnid);
    cASSERT0(txn, result.oldest_txnid >= lck->cached_oldest_txnid.weak);
    cASSERT0(txn, oldest_retired_pages >= lck->cached_oldest_retired.weak);
    atomic_store64(&lck->cached_oldest_txnid, result.oldest_txnid, mo_Relaxed);
    atomic_store64(&lck->cached_oldest_retired, oldest_retired_pages, mo_Relaxed);
    const meta_ptr_t recent = meta_recent(txn->env, &txn->wr.troika);
    const uint64_t reader_lag = recent.txnid - result.oldest_txnid;
    if (reader_lag > lck->pgops.gc_prof.max_reader_lag)
      lck->pgops.gc_prof.max_reader_lag = (reader_lag < UINT32_MAX) ? (uint32_t)reader_lag : UINT32_MAX;
    const uint64_t recent_retired = unaligned_peek_u64(4, recent.ptr_c->pages_retired);
    if (recent_retired > oldest_retired_pages) {
      const uint64_t retained_pages = recent_retired - oldest_retired_pages;
      if (retained_pages > lck->pgops.gc_prof.max_retained_pages)
        lck->pgops.gc_prof.max_retained_pages = (retained_pages < UINT32_MAX) ? (uint32_t)retained_pages : UINT32_MAX;
    }
  }
  return result;
}

pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page) {
  lck_t *const lck = env->lck;
retry:;
  const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
  for (size_t i = 0; i < snap_nreaders; ++i) {
    if (atomic_load_pid(&lck->rdt[i].pid, mo_AcquireRelease)) {
      /* jitter4testing(true); */
      const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
      const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid);
      if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) ||
                   snap_txnid != safe64_read(&lck->rdt[i].txnid)))
        goto retry;
      if (last_used_page < snap_pages && snap_txnid <= env->basal_txn->txnid)
        last_used_page = snap_pages;
    }
  }

  return last_used_page;
}

/* Find largest mvcc-snapshot still referenced by this process. */
pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest) {
  lck_t *const lck = env->lck;
  const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
  for (size_t i = 0; i < snap_nreaders; ++i) {
  retry:
    if (atomic_load_pid(&lck->rdt[i].pid, mo_AcquireRelease) == env->registered_reader_pid) {
      /* jitter4testing(true); */
      const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
      const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid);
      if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) ||
                   snap_txnid != safe64_read(&lck->rdt[i].txnid)))
        goto retry;
      if (largest < snap_pages &&
          atomic_load64(&lck->cached_oldest_txnid, mo_AcquireRelease) <=
              /* ignore pending updates */ snap_txnid &&
          snap_txnid <= MAX_TXNID)
        largest = snap_pages;
    }
  }
  return largest;
}

static bool pid_insert(mdbx_pid_t *list, mdbx_pid_t pid) {
  /* binary search of pid in list */
  size_t base = 0;
  size_t cursor = 1;
  int32_t val = 0;
  size_t n = /* length */ list[0];

  while (n > 0) {
    size_t pivot = n >> 1;
    cursor = base + pivot + 1;
    val = pid - list[cursor];

    if (val < 0) {
      n = pivot;
    } else if (val > 0) {
      base = cursor;
      n -= pivot + 1;
    } else {
      /* found, so it's a duplicate */
      return false;
    }
  }

  if (val > 0)
    ++cursor;

  list[0]++;
  for (n = list[0]; n > cursor; n--)
    list[n] = list[n - 1];
  list[n] = pid;
  return true;
}

__cold int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, int *dead) {
  int rc = check_env(env, true);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  eASSERT0(env, rdt_locked >= 0);
  lck_t *const lck = env->lck_mmap.lck;
  if (unlikely(lck == nullptr)) {
    /* exclusive mode */
    if (dead)
      *dead = 0;
    return MDBX_SUCCESS;
  }

  const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
  mdbx_pid_t pidsbuf_onstask[142];
  mdbx_pid_t *const pids = (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask))
                               ? pidsbuf_onstask
                               : osal_malloc((snap_nreaders + 1) * sizeof(mdbx_pid_t));
  if (unlikely(!pids))
    return MDBX_ENOMEM;

  pids[0] = 0;
  int count = 0;
  for (size_t i = 0; i < snap_nreaders; i++) {
    const mdbx_pid_t pid = atomic_load_pid(&lck->rdt[i].pid, mo_AcquireRelease);
    if (pid == 0)
      continue /* skip empty */;
    if (pid == env->pid)
      continue /* skip self */;
    if (!pid_insert(pids, pid))
      continue /* such pid already processed */;

    int err = lck_rpid_check(env, pid);
    if (err == MDBX_RESULT_TRUE)
      continue /* reader is live */;

    if (err != MDBX_SUCCESS) {
      rc = err;
      break /* lck_rpid_check() failed */;
    }

    /* stale reader found */
    if (!rdt_locked) {
      err = lck_rdt_lock(env);
      if (MDBX_IS_ERROR(err)) {
        rc = err;
        break;
      }

      rdt_locked = -1;
      if (err == MDBX_RESULT_TRUE) {
        /* mutex recovered, the mdbx_ipclock_failed() checked all readers */
        rc = MDBX_RESULT_TRUE;
        break;
      }

      /* a other process may have clean and reused slot, recheck */
      if (lck->rdt[i].pid.weak != (size_t)pid)
        continue;

      err = lck_rpid_check(env, pid);
      if (MDBX_IS_ERROR(err)) {
        rc = err;
        break;
      }

      if (err != MDBX_SUCCESS)
        continue /* the race with other process, slot reused */;
    }

    /* clean it */
    for (size_t ii = i; ii < snap_nreaders; ii++) {
      if (lck->rdt[ii].pid.weak == (size_t)pid) {
        DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, lck->rdt[ii].txnid.weak);
        atomic_store32(&lck->rdt[ii].pid, 0, mo_Relaxed);
        atomic_store32(&lck->rdt_refresh_flag, true, mo_AcquireRelease);
        count++;
      }
    }
  }

  if (likely(!MDBX_IS_ERROR(rc)))
    atomic_store64(&lck->readers_check_timestamp, osal_monotime(), mo_Relaxed);

  if (rdt_locked < 0)
    lck_rdt_unlock(env);

  if (pids != pidsbuf_onstask)
    osal_free(pids);

  if (dead)
    *dead = count;
  return rc;
}

__cold bool mvcc_kick_laggards(MDBX_txn *txn, const txnid_t straggler,
                               struct gc_reclaiming_obstacle *optional_obstacle) {
  DEBUG("DB size maxed out by reading #%" PRIaTXN, straggler);
  osal_memory_fence(mo_AcquireRelease, false);
  if (optional_obstacle) {
    optional_obstacle->pid = 0;
    optional_obstacle->tid = 0;
    optional_obstacle->txnid = 0;
  }

  MDBX_env *const env = txn->env;
  MDBX_hsr_func const callback = env->hsr_callback;
  orsi_rw_t orsi;
  bool notify_eof_of_loop = false;
  int retry = 0;

  do {
    env->lck->rdt_refresh_flag.weak = /* force refresh */ true;
    orsi = mvcc_shapshot_oldest_rw(txn);
    eASSERT0(env, orsi.oldest_txnid < env->basal_txn->txnid);
    eASSERT0(env, orsi.oldest_txnid >= straggler);
    eASSERT0(env, orsi.oldest_txnid >= env->lck->cached_oldest_txnid.weak);

    lck_t *const lck = env->lck_mmap.lck;
    if (orsi.oldest_txnid == orsi.steady_txnid || orsi.oldest_txnid > straggler || /* without-LCK mode */ !lck)
      break;

    if (MDBX_IS_ERROR(mvcc_cleanup_dead(env, false, nullptr)))
      break;

    reader_slot_t *stucked = nullptr;
    uint64_t stucked_retired = 0;
    uint64_t stucked_tid = 0;
    for (size_t i = 0; i < lck->rdt_length.weak; ++i) {
      mdbx_pid_t pid;
      reader_slot_t *const slot = &lck->rdt[i];
    retry:;
      const txnid_t slot_snap_mvcc = safe64_read(&slot->txnid);
      if (slot_snap_mvcc == orsi.oldest_txnid && (pid = atomic_load_pid(&slot->pid, mo_AcquireRelease)) != 0) {
        const uint64_t tid = safe64_read(&slot->tid);
        if (tid == MDBX_TID_TXN_PARKED) {
          /* Читающая транзакция была помечена владельцем как "припаркованная",
           * т.е. подлежащая асинхронному прерыванию, либо восстановлению
           * по активности читателя.
           *
           * Если первый CAS(slot->tid) будет успешным, то
           * safe64_reset_compare() безопасно очистит txnid, либо откажется
           * из-за того что читатель сбросил и/или перезапустил транзакцию.
           * При этом читатеть может не заметить вытестения, если приступит
           * к завершению транзакции. Все эти исходы нас устраивют.
           *
           * Если первый CAS(slot->tid) будет НЕ успешным, то значит читатеть
           * восстановил транзакцию, либо завершил её, либо даже освободил слот.
           */
          bool ousted =
#if MDBX_64BIT_CAS
              atomic_cas64(&slot->tid, MDBX_TID_TXN_PARKED, MDBX_TID_TXN_OUSTED);
#else
              atomic_cas32(&slot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)MDBX_TID_TXN_OUSTED);
#endif
          if (likely(ousted)) {
            ousted = safe64_reset_compare(&slot->txnid, slot_snap_mvcc);
            NOTICE("ousted-%s parked read-txn %" PRIaTXN ", pid %zu, tid 0x%" PRIx64, ousted ? "complete" : "half",
                   slot_snap_mvcc, (size_t)pid, tid);
            eASSERT0(env, ousted || safe64_read(&slot->txnid) > orsi.oldest_txnid);
            continue;
          }
          goto retry;
        }
        stucked_tid = tid;
        stucked_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed);
        stucked = slot;
      }
    }

    if (!stucked)
      break;

    mdbx_pid_t pid = atomic_load_pid(&stucked->pid, mo_AcquireRelease);
    if (safe64_read(&stucked->txnid) != straggler || safe64_read(&stucked->tid) != stucked_tid || !pid)
      continue;

    if (optional_obstacle) {
      optional_obstacle->pid = pid;
      optional_obstacle->tid = (mdbx_tid_t)((intptr_t)stucked_tid);
      optional_obstacle->txnid = straggler;
    }

    if (!callback)
      break;

    const meta_ptr_t recent = meta_recent(env, &env->txn->wr.troika);
    const txnid_t lag = (recent.txnid - straggler) / xMDBX_TXNID_STEP;
    const uint64_t recent_retired = unaligned_peek_u64(4, recent.ptr_c->pages_retired);
    const size_t space =
        (recent_retired > stucked_retired) ? pgno2bytes(env, (pgno_t)(recent_retired - stucked_retired)) : 0;
    int rc = callback(env, env->txn, pid, (mdbx_tid_t)((intptr_t)stucked_tid), straggler,
                      (lag < UINT_MAX) ? (unsigned)lag : UINT_MAX, space, retry);
    if (rc < 0) {
      /* hsr returned error and/or agree MDBX_MAP_FULL error */
      break;
    }

    if (rc > 0) {
      if (rc == 1) {
        /* hsr reported transaction (will be) aborted asynchronous */
        safe64_reset_compare(&stucked->txnid, straggler);
      } else {
        /* hsr reported reader process was killed and slot should be cleared */
        safe64_reset(&stucked->txnid, true);
        atomic_store64(&stucked->tid, MDBX_TID_TXN_OUSTED, mo_Relaxed);
        atomic_store32(&stucked->pid, 0, mo_AcquireRelease);
      }
    } else if (!notify_eof_of_loop) {
#if MDBX_ENABLE_PROFGC
      env->lck->pgops.gc_prof.kicks += 1;
#endif /* MDBX_ENABLE_PROFGC */
      notify_eof_of_loop = true;
    }

  } while (++retry < INT_MAX);

  if (notify_eof_of_loop) {
    /* notify end of hsr-loop */
    const txnid_t turn = orsi.oldest_txnid - straggler;
    if (turn)
      NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN, straggler, orsi.oldest_txnid, turn);
    callback(env, env->txn, 0, 0, straggler, (turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry);
  }
  return orsi.oldest_txnid > straggler;
}

__hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key) {
  page_t *mp = mc->pg[mc->top];
  MDBX_ANALYSIS_ASSUME(key != nullptr);
  DKBUF_DEBUG;
  DEBUG("add to leaf2-%spage %" PRIaPGNO " index %zi, "
        " key size %" PRIuPTR " [%s]",
        is_subpage(mp) ? "sub-" : "", mp->pgno, indx, key ? key->iov_len : 0, DKEY_DEBUG(key));

  cASSERT0(mc, key);
  cASSERT0(mc, page_type_compat(mp) == (P_LEAF | P_DUPFIX));
  const size_t ksize = mc->tree->dupfix_size;
  cASSERT0(mc, ksize == key->iov_len);
  const size_t nkeys = page_numkeys(mp);
  cASSERT0(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0);

  /* Just using these for counting */
  const intptr_t lower = mp->lower + sizeof(indx_t);
  const intptr_t upper = mp->upper - (ksize - sizeof(indx_t));
  if (unlikely(lower > upper)) {
    mc->txn->flags |= MDBX_TXN_ERROR;
    return MDBX_PAGE_FULL;
  }
  mp->lower = (indx_t)lower;
  mp->upper = (indx_t)upper;

  void *const ptr = page_dupfix_ptr(mp, indx, ksize);
  cASSERT0(mc, nkeys >= indx);
  const size_t diff = nkeys - indx;
  if (likely(diff > 0))
    /* Move higher keys up one slot. */
    memmove(ptr_disp(ptr, ksize), ptr, diff * ksize);
  /* insert new key */
  memcpy(ptr, key->iov_base, ksize);

  cASSERT0(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0);
  return MDBX_SUCCESS;
}

int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno) {
  page_t *mp = mc->pg[mc->top];
  DKBUF_DEBUG;
  DEBUG("add to branch-%spage %" PRIaPGNO " index %zi, node-pgno %" PRIaPGNO " key size %" PRIuPTR " [%s]",
        is_subpage(mp) ? "sub-" : "", mp->pgno, indx, pgno, key ? key->iov_len : 0, DKEY_DEBUG(key));

  cASSERT0(mc, page_type(mp) == P_BRANCH);
  cASSERT0(mc, mp->txnid >= mc->txn->front_txnid);
  STATIC_ASSERT(NODESIZE % 2 == 0);

  /* Adjust free space offsets. */
  const size_t branch_bytes = branch_size(mc->txn->env, key);
  const intptr_t lower = mp->lower + sizeof(indx_t);
  const intptr_t upper = mp->upper - (branch_bytes - sizeof(indx_t));
  if (unlikely(lower > upper)) {
    mc->txn->flags |= MDBX_TXN_ERROR;
    return MDBX_PAGE_FULL;
  }

  /* Move higher pointers up one slot. */
  const size_t nkeys = page_numkeys(mp);
  cASSERT0(mc, nkeys >= indx);
  for (size_t i = nkeys; i > indx; --i)
    mp->entries[i] = mp->entries[i - 1];

  mp->lower = (indx_t)lower;
  mp->entries[indx] = mp->upper = (indx_t)upper;

  /* Write the node data. */
  node_t *node = page_node(mp, indx);
  node_set_pgno(node, pgno);
  node_set_flags(node, 0);
  UNALIGNED_POKE_8(node, node_t, extra, 0);
  node_set_ks(node, 0);
  if (likely(key != nullptr)) {
    node_set_ks(node, key->iov_len);
    memcpy(node_key(node), key->iov_base, key->iov_len);
  }
  return MDBX_SUCCESS;
}

__hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data,
                                            unsigned flags) {
  MDBX_ANALYSIS_ASSUME(key != nullptr);
  MDBX_ANALYSIS_ASSUME(data != nullptr);
  page_t *mp = mc->pg[mc->top];
  DKBUF_DEBUG;
  DEBUG("add to leaf-%spage %" PRIaPGNO " index %zi, data size %" PRIuPTR " key size %" PRIuPTR " [%s]",
        is_subpage(mp) ? "sub-" : "", mp->pgno, indx, data ? data->iov_len : 0, key ? key->iov_len : 0,
        DKEY_DEBUG(key));
  cASSERT0(mc, key != nullptr && data != nullptr);
  cASSERT0(mc, page_type_compat(mp) == P_LEAF);
  page_t *largepage = nullptr;

  size_t node_bytes;
  if (unlikely(flags & N_BIG)) {
    /* Data already on large/overflow page. */
    STATIC_ASSERT(sizeof(pgno_t) % 2 == 0);
    node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
    cASSERT0(mc, page_room(mp) >= node_bytes);
  } else if (unlikely(node_size(key, data) > mc->txn->env->leaf_nodemax)) {
    /* Put data on large/overflow page. */
    if (unlikely(mc->tree->flags & MDBX_DUPSORT)) {
      ERROR("Unexpected target %s flags 0x%x for large data-item", "dupsort-db", mc->tree->flags);
      return MDBX_PROBLEM;
    }
    if (unlikely(flags & (N_DUP | N_TREE))) {
      ERROR("Unexpected target %s flags 0x%x for large data-item", "node", flags);
      return MDBX_PROBLEM;
    }
    cASSERT0(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data));
    const pgno_t ovpages = largechunk_npages(mc->txn->env, data->iov_len);
    const pgr_t npr = page_new_large(mc, ovpages);
    if (unlikely(npr.err != MDBX_SUCCESS))
      return npr.err;
    largepage = npr.page;
    DEBUG("allocated %u large/overflow page(s) %" PRIaPGNO "for %" PRIuPTR " data bytes", largepage->pages,
          largepage->pgno, data->iov_len);
    flags |= N_BIG;
    node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
    cASSERT0(mc, node_bytes == leaf_size(mc->txn->env, key, data));
  } else {
    cASSERT0(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data));
    node_bytes = node_size(key, data) + sizeof(indx_t);
    cASSERT0(mc, node_bytes == leaf_size(mc->txn->env, key, data));
  }

  /* Move higher pointers up one slot. */
  const size_t nkeys = page_numkeys(mp);
  cASSERT0(mc, nkeys >= indx);
  for (size_t i = nkeys; i > indx; --i)
    mp->entries[i] = mp->entries[i - 1];

  /* Adjust free space offsets. */
  const intptr_t lower = mp->lower + sizeof(indx_t);
  const intptr_t upper = mp->upper - (node_bytes - sizeof(indx_t));
  if (unlikely(lower > upper)) {
    mc->txn->flags |= MDBX_TXN_ERROR;
    return MDBX_PAGE_FULL;
  }
  mp->lower = (indx_t)lower;
  mp->entries[indx] = mp->upper = (indx_t)upper;

  /* Write the node data. */
  node_t *node = page_node(mp, indx);
  node_set_ks(node, key->iov_len);
  node_set_flags(node, (uint8_t)flags);
  UNALIGNED_POKE_8(node, node_t, extra, 0);
  node_set_ds(node, data->iov_len);
  memcpy(node_key(node), key->iov_base, key->iov_len);

  void *nodedata = node_data(node);
  if (likely(largepage == nullptr)) {
    if (unlikely(flags & N_BIG)) {
      memcpy(nodedata, data->iov_base, sizeof(pgno_t));
      return MDBX_SUCCESS;
    }
  } else {
    poke_pgno(nodedata, largepage->pgno);
    nodedata = page2payload(largepage);
  }
  if (unlikely(flags & MDBX_RESERVE))
    data->iov_base = nodedata;
  else if (likely(data->iov_len /* to avoid UBSAN traps */))
    memcpy(nodedata, data->iov_base, data->iov_len);
  return MDBX_SUCCESS;
}

__hot void node_del(MDBX_cursor *mc, size_t ksize) {
  page_t *mp = mc->pg[mc->top];
  const size_t hole = mc->ki[mc->top];
  const size_t nkeys = page_numkeys(mp);

  DEBUG("delete node %zu on %s page %" PRIaPGNO, hole, is_leaf(mp) ? "leaf" : "branch", mp->pgno);
  cASSERT0(mc, hole < nkeys);

  if (is_dupfix_leaf(mp)) {
    cASSERT0(mc, ksize >= sizeof(indx_t));
    size_t diff = nkeys - 1 - hole;
    void *const base = page_dupfix_ptr(mp, hole, ksize);
    if (diff)
      memmove(base, ptr_disp(base, ksize), diff * ksize);
    cASSERT0(mc, mp->lower >= sizeof(indx_t));
    mp->lower -= sizeof(indx_t);
    cASSERT0(mc, (size_t)UINT16_MAX - mp->upper >= ksize - sizeof(indx_t));
    mp->upper += (indx_t)(ksize - sizeof(indx_t));
    cASSERT0(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0);
    return;
  }

  node_t *node = page_node(mp, hole);
  cASSERT0(mc, !is_branch(mp) || hole || node_ks(node) == 0);
  size_t hole_size = NODESIZE + node_ks(node);
  if (is_leaf(mp))
    hole_size += (node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node);
  hole_size = EVEN_CEIL(hole_size);

  const indx_t hole_offset = mp->entries[hole];
  size_t r, w;
  for (r = w = 0; r < nkeys; r++)
    if (r != hole)
      mp->entries[w++] = (mp->entries[r] < hole_offset) ? mp->entries[r] + (indx_t)hole_size : mp->entries[r];

  void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ);
  memmove(ptr_disp(base, hole_size), base, hole_offset - mp->upper);

  cASSERT0(mc, mp->lower >= sizeof(indx_t));
  mp->lower -= sizeof(indx_t);
  cASSERT0(mc, (size_t)UINT16_MAX - mp->upper >= hole_size);
  mp->upper += (indx_t)hole_size;

  if (CHECKS2_ENABLED()) {
    const uint8_t checking = mc->checking;
    mc->checking |= z_updating;
    const int page_check_err = page_check(mc, mp);
    mc->checking = checking;
    cASSERT0(mc, page_check_err == MDBX_SUCCESS);
  }
}

__noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) {
  cASSERT0(mc, node_flags(node) == N_BIG && data->iov_len == node_ds(node));

  pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid);
  if (unlikely((lp.err != MDBX_SUCCESS))) {
    DEBUG("read large/overflow page %" PRIaPGNO " failed", node_largedata_pgno(node));
    return lp.err;
  }

  cASSERT0(mc, page_type(lp.page) == P_LARGE);
  data->iov_base = page2payload(lp.page);
  if (!MDBX_DISABLE_VALIDATION) {
    const MDBX_env *env = mc->txn->env;
    const size_t dsize = data->iov_len;
    const unsigned npages = largechunk_npages(env, dsize);
    if (unlikely(lp.page->pages < npages))
      return bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize);
  }
  return MDBX_SUCCESS;
}

node_t *node_shrink(page_t *mp, size_t indx, node_t *node) {
  ASSERT(node == page_node(mp, indx));
  page_t *sp = (page_t *)node_data(node);
  ASSERT(is_subpage(sp) && page_numkeys(sp) > 0);
  const size_t delta = EVEN_FLOOR(page_room(sp) /* avoid the node uneven-sized */);
  if (unlikely(delta) == 0)
    return node;

  /* Prepare to shift upward, set len = length(subpage part to shift) */
  size_t nsize = node_ds(node) - delta, len = nsize;
  ASSERT(nsize % 1 == 0);
  if (!is_dupfix_leaf(sp)) {
    len = PAGEHDRSZ;
    page_t *xp = ptr_disp(sp, delta); /* destination subpage */
    for (intptr_t i = page_numkeys(sp); --i >= 0;) {
      ASSERT(sp->entries[i] >= delta);
      xp->entries[i] = (indx_t)(sp->entries[i] - delta);
    }
  }
  ASSERT(sp->upper >= sp->lower + delta);
  sp->upper -= (indx_t)delta;
  sp->pgno = mp->pgno;
  node_set_ds(node, nsize);

  /* Shift <lower nodes...initial part of subpage> upward */
  void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ);
  memmove(ptr_disp(base, delta), base, ptr_dist(sp, base) + len);

  const size_t pivot = mp->entries[indx];
  for (intptr_t i = page_numkeys(mp); --i >= 0;) {
    if (mp->entries[i] <= pivot) {
      ASSERT((size_t)UINT16_MAX - mp->entries[i] >= delta);
      mp->entries[i] += (indx_t)delta;
    }
  }
  ASSERT((size_t)UINT16_MAX - mp->upper >= delta);
  mp->upper += (indx_t)delta;

  return ptr_disp(node, delta);
}

#if defined(_WIN32) || defined(_WIN64)

#include <psapi.h>
#include <winioctl.h>

#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG)
#include <crtdbg.h>
#endif

/* Map a result from an NTAPI call to WIN32 error code. */
static int osal_ntstatus2errcode(NTSTATUS status) {
  DWORD dummy;
  OVERLAPPED ov;
  memset(&ov, 0, sizeof(ov));
  ov.Internal = status;
  /* Zap: '_Param_(1)' could be '0' */
  MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6387);
  return GetOverlappedResult(nullptr, &ov, &dummy, FALSE) ? MDBX_SUCCESS : (int)GetLastError();
}

MDBX_INTERNAL int osal_waitstatus2errcode(DWORD result) {
  switch (result) {
  case WAIT_OBJECT_0:
    return MDBX_SUCCESS;
  case WAIT_FAILED:
    return (int)GetLastError();
  case WAIT_ABANDONED:
    return ERROR_ABANDONED_WAIT_0;
  case WAIT_IO_COMPLETION:
    return ERROR_USER_APC;
  case WAIT_TIMEOUT:
    return ERROR_TIMEOUT;
  default:
    return osal_ntstatus2errcode(result);
  }
}

/* We use native NT APIs to setup the memory map, so that we can
 * let the DB file grow incrementally instead of always preallocating
 * the full size. These APIs are defined in <wdm.h> and <ntifs.h>
 * but those headers are meant for driver-level development and
 * conflict with the regular user-level headers, so we explicitly
 * declare them here. Using these APIs also means we must link to
 * ntdll.dll, which is not linked by default in user code. */

extern NTSTATUS NTAPI NtCreateSection(OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess,
                                      IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes,
                                      IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection,
                                      IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle);

typedef struct _SECTION_BASIC_INFORMATION {
  ULONG Unknown;
  ULONG SectionAttributes;
  LARGE_INTEGER SectionSize;
} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION;

extern NTSTATUS NTAPI NtMapViewOfSection(IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress,
                                         IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize,
                                         IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize,
                                         IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType,
                                         IN ULONG Win32Protect);

extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, IN OPTIONAL PVOID BaseAddress);

/* Zap: Inconsistent annotation for 'NtClose'... */
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(28251)
extern NTSTATUS NTAPI NtClose(HANDLE Handle);

extern NTSTATUS NTAPI NtAllocateVirtualMemory(IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits,
                                              IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect);

extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, IN PVOID *BaseAddress, IN OUT PSIZE_T RegionSize,
                                          IN ULONG FreeType);

#ifndef WOF_CURRENT_VERSION
typedef struct _WOF_EXTERNAL_INFO {
  DWORD Version;
  DWORD Provider;
} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO;
#endif /* WOF_CURRENT_VERSION */

#ifndef WIM_PROVIDER_CURRENT_VERSION
#define WIM_PROVIDER_HASH_SIZE 20

typedef struct _WIM_PROVIDER_EXTERNAL_INFO {
  DWORD Version;
  DWORD Flags;
  LARGE_INTEGER DataSourceId;
  BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE];
} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO;
#endif /* WIM_PROVIDER_CURRENT_VERSION */

#ifndef FILE_PROVIDER_CURRENT_VERSION
typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 {
  ULONG Version;
  ULONG Algorithm;
  ULONG Flags;
} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1;
#endif /* FILE_PROVIDER_CURRENT_VERSION */

#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED
#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL)
#endif
#ifndef STATUS_INVALID_DEVICE_REQUEST
#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L)
#endif
#ifndef STATUS_NOT_SUPPORTED
#define STATUS_NOT_SUPPORTED ((NTSTATUS)0xC00000BBL)
#endif

#ifndef FILE_DEVICE_FILE_SYSTEM
#define FILE_DEVICE_FILE_SYSTEM 0x00000009
#endif

#ifndef FSCTL_GET_EXTERNAL_BACKING
#define FSCTL_GET_EXTERNAL_BACKING CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS)
#endif

#ifndef ERROR_NOT_CAPABLE
#define ERROR_NOT_CAPABLE 775L
#endif

#endif /* _WIN32 || _WIN64 */

/*----------------------------------------------------------------------------*/

#if defined(__ANDROID_API__)
__extern_C void __assert2(const char *file, int line, const char *function, const char *msg) __noreturn;
#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion)

#elif defined(__UCLIBC__)
MDBX_NORETURN __extern_C void __assert(const char *, const char *, unsigned, const char *)
#ifdef __THROW
    __THROW
#else
    __nothrow
#endif /* __THROW */
    ;
#define __assert_fail(assertion, file, line, function) __assert(assertion, file, line, function)

#elif _POSIX_C_SOURCE > 200212 &&                                                                                      \
    /* workaround for avoid musl libc wrong prototype */ (defined(__GLIBC__) || defined(__GNU_LIBRARY__))
/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */
MDBX_NORETURN __extern_C void __assert_fail(const char *assertion, const char *file, unsigned line,
                                            const char *function)
#ifdef __THROW
    __THROW
#else
    __nothrow
#endif /* __THROW */
    ;

#elif defined(__APPLE__) || defined(__MACH__)
__extern_C void __assert_rtn(const char *function, const char *file, int line, const char *assertion) /* __nothrow */
#ifdef __dead2
    __dead2
#else
    MDBX_NORETURN
#endif /* __dead2 */
#ifdef __disable_tail_calls
    __disable_tail_calls
#endif /* __disable_tail_calls */
    ;

#define __assert_fail(assertion, file, line, function) __assert_rtn(function, file, line, assertion)
#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
MDBX_NORETURN __extern_C void __assert_c99(const char *assection, const char *file, int line, const char *function);
#define __assert_fail(assertion, file, line, function) __assert_c99(assertion, file, line, function)
#elif defined(__OpenBSD__)
__extern_C __dead void __assert2(const char *file, int line, const char *function,
                                 const char *assertion) /* __nothrow */;
#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion)
#elif defined(__NetBSD__)
__extern_C __dead void __assert13(const char *file, int line, const char *function,
                                  const char *assertion) /* __nothrow */;
#define __assert_fail(assertion, file, line, function) __assert13(file, line, function, assertion)
#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__)
__extern_C void __assert(const char *function, const char *file, int line, const char *assertion) /* __nothrow */
#ifdef __dead2
    __dead2
#else
    MDBX_NORETURN
#endif /* __dead2 */
#ifdef __disable_tail_calls
    __disable_tail_calls
#endif /* __disable_tail_calls */
    ;
#define __assert_fail(assertion, file, line, function) __assert(function, file, line, assertion)

#endif /* __assert_fail */

__cold void osal_panic(const char *msg, const char *func, unsigned line) {
  while (1) {
#if defined(_WIN32) || defined(_WIN64)
#if defined(_DEBUG) && !MDBX_WITHOUT_MSVC_CRT
    _CrtDbgReport(_CRT_ASSERT, func, line, "libmdbx", "assertion failed: %s", msg);
#else
    char *message = nullptr;
    const int num = osal_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u\r\n", msg, func, line);
    if (num < 1 || !message)
      message = "\r\n<asprintf() failed>\r\n";
    OutputDebugStringA(message);
#endif
    if (IsDebuggerPresent())
      DebugBreak();
    FatalExit(STATUS_ASSERTION_FAILURE);
#else
    __assert_fail(msg, "libmdbx", line, func);
    abort();
#endif
  }
}

/*----------------------------------------------------------------------------*/

#ifndef osal_vasprintf
int osal_vasprintf(char **strp, const char *fmt, va_list ap) {
  va_list ones;
  va_copy(ones, ap);
  const int needed = vsnprintf(nullptr, 0, fmt, ones);
  va_end(ones);

  if (unlikely(needed < 0 || needed >= INT_MAX)) {
    *strp = nullptr;
    return needed;
  }

  *strp = osal_malloc(needed + (size_t)1);
  if (unlikely(*strp == nullptr)) {
#if defined(_WIN32) || defined(_WIN64)
    SetLastError(MDBX_ENOMEM);
#else
    errno = MDBX_ENOMEM;
#endif
    return -1;
  }

  const int actual = vsnprintf(*strp, needed + (size_t)1, fmt, ap);
  ASSERT(actual == needed);
  if (unlikely(actual < 0)) {
    osal_free(*strp);
    *strp = nullptr;
  }
  return actual;
}
#endif /* osal_vasprintf */

#ifndef osal_asprintf
int osal_asprintf(char **strp, const char *fmt, ...) {
  va_list ap;
  va_start(ap, fmt);
  const int rc = osal_vasprintf(strp, fmt, ap);
  va_end(ap);
  return rc;
}
#endif /* osal_asprintf */

#ifndef osal_memalign_alloc
int osal_memalign_alloc(size_t alignment, size_t bytes, void **result) {
  ASSERT(is_powerof2(alignment) && alignment >= sizeof(void *));
#if defined(_WIN32) || defined(_WIN64)
  (void)alignment;
  *result = VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
  return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */;
#elif defined(_ISOC11_SOURCE)
  *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment));
  return *result ? MDBX_SUCCESS : errno;
#elif _POSIX_VERSION >= 200112L && (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17)
  *result = nullptr;
  return posix_memalign(result, alignment, bytes);
#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L
  *result = memalign(alignment, bytes);
  return *result ? MDBX_SUCCESS : errno;
#else
#error FIXME
#endif
}
#endif /* osal_memalign_alloc */

#ifndef osal_memalign_free
void osal_memalign_free(void *ptr) {
#if defined(_WIN32) || defined(_WIN64)
  VirtualFree(ptr, 0, MEM_RELEASE);
#else
  osal_free(ptr);
#endif
}
#endif /* osal_memalign_free */

#ifndef osal_strdup
char *osal_strdup(const char *str) {
  if (!str)
    return nullptr;
  size_t bytes = strlen(str) + 1;
  char *dup = osal_malloc(bytes);
  if (dup)
    memcpy(dup, str, bytes);
  return dup;
}
#endif /* osal_strdup */

/*----------------------------------------------------------------------------*/

int osal_condpair_init(osal_condpair_t *condpair) {
  int rc;
  memset(condpair, 0, sizeof(osal_condpair_t));
#if defined(_WIN32) || defined(_WIN64)
  if (!(condpair->mutex = CreateMutexW(nullptr, FALSE, nullptr))) {
    rc = (int)GetLastError();
    goto bailout_mutex;
  }
  if (!(condpair->event[0] = CreateEventW(nullptr, FALSE, FALSE, nullptr))) {
    rc = (int)GetLastError();
    goto bailout_event;
  }
  if ((condpair->event[1] = CreateEventW(nullptr, FALSE, FALSE, nullptr)))
    return MDBX_SUCCESS;

  rc = (int)GetLastError();
  (void)CloseHandle(condpair->event[0]);
bailout_event:
  (void)CloseHandle(condpair->mutex);
#else
  rc = pthread_mutex_init(&condpair->mutex, nullptr);
  if (unlikely(rc != 0))
    goto bailout_mutex;
  rc = pthread_cond_init(&condpair->cond[0], nullptr);
  if (unlikely(rc != 0))
    goto bailout_cond;
  rc = pthread_cond_init(&condpair->cond[1], nullptr);
  if (likely(rc == 0))
    return MDBX_SUCCESS;

  (void)pthread_cond_destroy(&condpair->cond[0]);
bailout_cond:
  (void)pthread_mutex_destroy(&condpair->mutex);
#endif
bailout_mutex:
  memset(condpair, 0, sizeof(osal_condpair_t));
  return rc;
}

int osal_condpair_destroy(osal_condpair_t *condpair) {
#if defined(_WIN32) || defined(_WIN64)
  int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
  rc = CloseHandle(condpair->event[0]) ? rc : (int)GetLastError();
  rc = CloseHandle(condpair->event[1]) ? rc : (int)GetLastError();
#else
  int err, rc = pthread_mutex_destroy(&condpair->mutex);
  rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc;
  rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc;
#endif
  memset(condpair, 0, sizeof(osal_condpair_t));
  return rc;
}

int osal_condpair_lock(osal_condpair_t *condpair) {
#if defined(_WIN32) || defined(_WIN64)
  DWORD code = WaitForSingleObject(condpair->mutex, INFINITE);
  return osal_waitstatus2errcode(code);
#else
  return osal_pthread_mutex_lock(&condpair->mutex);
#endif
}

int osal_condpair_unlock(osal_condpair_t *condpair) {
#if defined(_WIN32) || defined(_WIN64)
  return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
#else
  return pthread_mutex_unlock(&condpair->mutex);
#endif
}

int osal_condpair_signal(osal_condpair_t *condpair, bool part) {
#if defined(_WIN32) || defined(_WIN64)
  return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : (int)GetLastError();
#else
  return pthread_cond_signal(&condpair->cond[part]);
#endif
}

int osal_condpair_wait(osal_condpair_t *condpair, bool part) {
#if defined(_WIN32) || defined(_WIN64)
  DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], INFINITE, FALSE);
  if (code == WAIT_OBJECT_0) {
    code = WaitForSingleObject(condpair->mutex, INFINITE);
    if (code == WAIT_OBJECT_0)
      return MDBX_SUCCESS;
  }
  return osal_waitstatus2errcode(code);
#else
  return pthread_cond_wait(&condpair->cond[part], &condpair->mutex);
#endif
}

/*----------------------------------------------------------------------------*/

int osal_fastmutex_init(osal_fastmutex_t *fastmutex) {
#if defined(_WIN32) || defined(_WIN64)
  InitializeCriticalSection(fastmutex);
  return MDBX_SUCCESS;
#elif MDBX_CHECKING > 1
  pthread_mutexattr_t ma;
  int rc = pthread_mutexattr_init(&ma);
  if (likely(!rc)) {
    rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
    if (likely(!rc) || rc == ENOTSUP)
      rc = pthread_mutex_init(fastmutex, &ma);
    pthread_mutexattr_destroy(&ma);
  }
  return rc;
#else
  return pthread_mutex_init(fastmutex, nullptr);
#endif
}

int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex) {
#if defined(_WIN32) || defined(_WIN64)
  DeleteCriticalSection(fastmutex);
  return MDBX_SUCCESS;
#else
  return pthread_mutex_destroy(fastmutex);
#endif
}

int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex) {
#if defined(_WIN32) || defined(_WIN64)
  __try {
    EnterCriticalSection(fastmutex);
  } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
                  ? EXCEPTION_EXECUTE_HANDLER
                  : EXCEPTION_CONTINUE_SEARCH) {
    return MDBX_EDEADLK;
  }
  return MDBX_SUCCESS;
#else
  return osal_pthread_mutex_lock(fastmutex);
#endif
}

int osal_fastmutex_release(osal_fastmutex_t *fastmutex) {
#if defined(_WIN32) || defined(_WIN64)
  LeaveCriticalSection(fastmutex);
  return MDBX_SUCCESS;
#else
  return pthread_mutex_unlock(fastmutex);
#endif
}

/*----------------------------------------------------------------------------*/

#if defined(_WIN32) || defined(_WIN64)

int osal_mb2w(const char *const src, wchar_t **const pdst) {
  const size_t dst_wlen = MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, nullptr, 0);
  wchar_t *dst = *pdst;
  int rc = ERROR_INVALID_NAME;
  if (unlikely(dst_wlen < 2 || dst_wlen > /* MAX_PATH */ INT16_MAX))
    goto bailout;

  dst = osal_realloc(dst, dst_wlen * sizeof(wchar_t));
  rc = MDBX_ENOMEM;
  if (unlikely(!dst))
    goto bailout;

  *pdst = dst;
  if (likely(dst_wlen == (size_t)MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, dst, (int)dst_wlen)))
    return MDBX_SUCCESS;

  rc = ERROR_INVALID_NAME;
bailout:
  if (*pdst) {
    osal_free(*pdst);
    *pdst = nullptr;
  }
  return rc;
}

#endif /* Windows */

/*----------------------------------------------------------------------------*/

#if defined(_WIN32) || defined(_WIN64)
#define ior_alignment_mask (ior->pagesize - 1)
#define ior_WriteFile_flag 1
#define OSAL_IOV_MAX (4096 / sizeof(ior_sgv_element))

static void ior_put_event(osal_ioring_t *ior, HANDLE event) {
  ASSERT(event && event != INVALID_HANDLE_VALUE && event != ior);
  ASSERT(ior->event_stack < ior->allocated);
  ior->event_pool[ior->event_stack] = event;
  ior->event_stack += 1;
}

static HANDLE ior_get_event(osal_ioring_t *ior) {
  ASSERT(ior->event_stack <= ior->allocated);
  if (ior->event_stack > 0) {
    ior->event_stack -= 1;
    ASSERT(ior->event_pool[ior->event_stack] != 0);
    return ior->event_pool[ior->event_stack];
  }
  return CreateEventW(nullptr, true, false, nullptr);
}

static void WINAPI ior_wocr(DWORD err, DWORD bytes, OVERLAPPED *ov) {
  osal_ioring_t *ior = ov->hEvent;
  ov->Internal = err;
  ov->InternalHigh = bytes;
  if (++ior->async_completed >= ior->async_waiting)
    SetEvent(ior->async_done);
}

#elif MDBX_HAVE_PWRITEV
#if defined(_SC_IOV_MAX) || defined(ENABLE_MEMCHECK)
static size_t osal_iov_max;
#define OSAL_IOV_MAX osal_iov_max
#else
#define OSAL_IOV_MAX IOV_MAX
#endif
#else
#undef OSAL_IOV_MAX
#endif /* OSAL_IOV_MAX */

int osal_ioring_create(osal_ioring_t *ior
#if defined(_WIN32) || defined(_WIN64)
                       ,
                       bool enable_direct, mdbx_filehandle_t overlapped_fd
#endif /* Windows */
) {
  memset(ior, 0, sizeof(osal_ioring_t));

#if defined(_WIN32) || defined(_WIN64)
  ior->overlapped_fd = overlapped_fd;
  ior->direct = enable_direct && overlapped_fd;
  ior->pagesize = globals.sys_pagesize;
  ior->pagesize_ln2 = globals.sys_pagesize_ln2;
  ior->async_done = ior_get_event(ior);
  if (!ior->async_done)
    return GetLastError();
#endif /* !Windows */

#if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX)
  ASSERT(osal_iov_max > 0);
#endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */

  ior->boundary = ptr_disp(ior->pool, ior->allocated);
  return MDBX_SUCCESS;
}

static inline size_t ior_offset(const ior_item_t *item) {
#if defined(_WIN32) || defined(_WIN64)
  return item->ov.Offset |
         (size_t)((sizeof(size_t) > sizeof(item->ov.Offset)) ? (uint64_t)item->ov.OffsetHigh << 32 : 0);
#else
  return item->offset;
#endif /* !Windows */
}

static inline ior_item_t *ior_next(ior_item_t *item, size_t sgvcnt) {
#if defined(ior_sgv_element)
  ASSERT(sgvcnt > 0);
  return (ior_item_t *)ptr_disp(item, sizeof(ior_item_t) - sizeof(ior_sgv_element) + sizeof(ior_sgv_element) * sgvcnt);
#else
  ASSERT(sgvcnt == 1);
  (void)sgvcnt;
  return item + 1;
#endif
}

int osal_ioring_add(osal_ioring_t *ior, const size_t offset, void *data, const size_t bytes) {
  ASSERT(bytes && data);
  ASSERT(bytes % MDBX_MIN_PAGESIZE == 0 && bytes <= MAX_WRITE);
  ASSERT(offset % MDBX_MIN_PAGESIZE == 0 && offset + (uint64_t)bytes <= MAX_MAPSIZE);

#if defined(_WIN32) || defined(_WIN64)
  const unsigned segments = (unsigned)(bytes >> ior->pagesize_ln2);
  const bool use_gather = ior->direct && ior->overlapped_fd && ior->slots_left >= segments;
#endif /* Windows */

  ior_item_t *item = ior->pool;
  if (likely(ior->last)) {
    item = ior->last;
    if (unlikely(ior_offset(item) + ior_last_bytes(ior, item) == offset) &&
        likely(ior_last_bytes(ior, item) + bytes <= MAX_WRITE)) {
#if defined(_WIN32) || defined(_WIN64)
      if (use_gather &&
          ((bytes | (uintptr_t)data | ior->last_bytes | (uintptr_t)(uint64_t)item->sgv[0].Buffer) &
           ior_alignment_mask) == 0 &&
          ior->last_sgvcnt + (size_t)segments < OSAL_IOV_MAX) {
        ASSERT(ior->overlapped_fd);
        ASSERT((item->single.iov_len & ior_WriteFile_flag) == 0);
        ASSERT(item->sgv[ior->last_sgvcnt].Buffer == 0);
        ior->last_bytes += bytes;
        size_t i = 0;
        do {
          item->sgv[ior->last_sgvcnt + i].Buffer = PtrToPtr64(data);
          data = ptr_disp(data, ior->pagesize);
        } while (++i < segments);
        ior->slots_left -= segments;
        item->sgv[ior->last_sgvcnt += segments].Buffer = 0;
        ASSERT((item->single.iov_len & ior_WriteFile_flag) == 0);
        return MDBX_SUCCESS;
      }
      const void *end = ptr_disp(item->single.iov_base, item->single.iov_len - ior_WriteFile_flag);
      if (unlikely(end == data)) {
        ASSERT((item->single.iov_len & ior_WriteFile_flag) != 0);
        item->single.iov_len += bytes;
        return MDBX_SUCCESS;
      }
#elif MDBX_HAVE_PWRITEV
      ASSERT((int)item->sgvcnt > 0);
      const void *end = ptr_disp(item->sgv[item->sgvcnt - 1].iov_base, item->sgv[item->sgvcnt - 1].iov_len);
      if (unlikely(end == data)) {
        item->sgv[item->sgvcnt - 1].iov_len += bytes;
        ior->last_bytes += bytes;
        return MDBX_SUCCESS;
      }
      if (likely(item->sgvcnt < OSAL_IOV_MAX)) {
        if (unlikely(ior->slots_left < 1))
          return MDBX_RESULT_TRUE;
        item->sgv[item->sgvcnt].iov_base = data;
        item->sgv[item->sgvcnt].iov_len = bytes;
        ior->last_bytes += bytes;
        item->sgvcnt += 1;
        ior->slots_left -= 1;
        return MDBX_SUCCESS;
      }
#else
      const void *end = ptr_disp(item->single.iov_base, item->single.iov_len);
      if (unlikely(end == data)) {
        item->single.iov_len += bytes;
        return MDBX_SUCCESS;
      }
#endif
    }
    item = ior_next(item, ior_last_sgvcnt(ior, item));
  }

  if (unlikely(ior->slots_left < 1))
    return MDBX_RESULT_TRUE;

  unsigned slots_used = 1;
#if defined(_WIN32) || defined(_WIN64)
  item->ov.Internal = item->ov.InternalHigh = 0;
  item->ov.Offset = (DWORD)offset;
  item->ov.OffsetHigh = HIGH_DWORD(offset);
  item->ov.hEvent = 0;
  if (!use_gather || ((bytes | (uintptr_t)(data)) & ior_alignment_mask) != 0 || segments > OSAL_IOV_MAX) {
    /* WriteFile() */
    item->single.iov_base = data;
    item->single.iov_len = bytes + ior_WriteFile_flag;
    ASSERT((item->single.iov_len & ior_WriteFile_flag) != 0);
  } else {
    /* WriteFileGather() */
    ASSERT(ior->overlapped_fd);
    item->sgv[0].Buffer = PtrToPtr64(data);
    for (size_t i = 1; i < segments; ++i) {
      data = ptr_disp(data, ior->pagesize);
      item->sgv[i].Buffer = PtrToPtr64(data);
    }
    item->sgv[slots_used = segments].Buffer = 0;
    ASSERT((item->single.iov_len & ior_WriteFile_flag) == 0);
  }
  ior->last_bytes = bytes;
  ior_last_sgvcnt(ior, item) = slots_used;
#elif MDBX_HAVE_PWRITEV
  item->offset = offset;
  item->sgv[0].iov_base = data;
  item->sgv[0].iov_len = bytes;
  ior->last_bytes = bytes;
  ior_last_sgvcnt(ior, item) = slots_used;
#else
  item->offset = offset;
  item->single.iov_base = data;
  item->single.iov_len = bytes;
#endif /* !Windows */
  ior->slots_left -= slots_used;
  ior->last = item;
  return MDBX_SUCCESS;
}

void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx,
                      void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)) {
  for (ior_item_t *item = ior->pool; item <= ior->last;) {
#if defined(_WIN32) || defined(_WIN64)
    size_t offset = ior_offset(item);
    char *data = item->single.iov_base;
    size_t bytes = item->single.iov_len - ior_WriteFile_flag;
    size_t i = 1;
    if (bytes & ior_WriteFile_flag) {
      data = Ptr64ToPtr(item->sgv[0].Buffer);
      bytes = ior->pagesize;
      /* Zap: Reading invalid data from 'item->sgv' */
      MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
      while (item->sgv[i].Buffer) {
        if (data + ior->pagesize != item->sgv[i].Buffer) {
          callback(ctx, offset, data, bytes);
          offset += bytes;
          data = Ptr64ToPtr(item->sgv[i].Buffer);
          bytes = 0;
        }
        bytes += ior->pagesize;
        ++i;
      }
    }
    ASSERT(bytes < MAX_WRITE);
    callback(ctx, offset, data, bytes);
#elif MDBX_HAVE_PWRITEV
    ASSERT(item->sgvcnt > 0);
    size_t offset = item->offset;
    size_t i = 0;
    do {
      callback(ctx, offset, item->sgv[i].iov_base, item->sgv[i].iov_len);
      offset += item->sgv[i].iov_len;
    } while (++i != item->sgvcnt);
#else
    const size_t i = 1;
    callback(ctx, item->offset, item->single.iov_base, item->single.iov_len);
#endif
    item = ior_next(item, i);
  }
}

osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) {
  osal_ioring_write_result_t r = {MDBX_SUCCESS, 0};

#if defined(_WIN32) || defined(_WIN64)
  HANDLE *const end_wait_for = ior->event_pool + ior->allocated +
                               /* был выделен один дополнительный элемент для async_done */ 1;
  HANDLE *wait_for = end_wait_for;
  LONG async_started = 0;
  for (ior_item_t *item = ior->pool; item <= ior->last;) {
    item->ov.Internal = STATUS_PENDING;
    size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag;
    r.wops += 1;
    if (bytes & ior_WriteFile_flag) {
      ASSERT(ior->overlapped_fd && fd == ior->overlapped_fd);
      bytes = ior->pagesize;
      /* Zap: Reading invalid data from 'item->sgv' */
      MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
      while (item->sgv[i].Buffer) {
        bytes += ior->pagesize;
        ++i;
      }
      ASSERT(bytes < MAX_WRITE);
      item->ov.hEvent = ior_get_event(ior);
      if (unlikely(!item->ov.hEvent)) {
      bailout_geterr:
        r.err = GetLastError();
      bailout_rc:
        ASSERT(r.err != MDBX_SUCCESS);
        CancelIo(fd);
        return r;
      }
      if (WriteFileGather(fd, item->sgv, (DWORD)bytes, nullptr, &item->ov)) {
        ASSERT(item->ov.Internal == 0 && WaitForSingleObject(item->ov.hEvent, 0) == WAIT_OBJECT_0);
        ior_put_event(ior, item->ov.hEvent);
        item->ov.hEvent = 0;
      } else {
        r.err = (int)GetLastError();
        if (unlikely(r.err != ERROR_IO_PENDING)) {
          void *data = Ptr64ToPtr(item->sgv[0].Buffer);
          ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
                " offset %" PRId64 ", err %d",
                "WriteFileGather", fd, __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno,
                bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err);
          goto bailout_rc;
        }
        ASSERT(wait_for > ior->event_pool + ior->event_stack);
        *--wait_for = item->ov.hEvent;
      }
    } else if (fd == ior->overlapped_fd) {
      ASSERT(bytes < MAX_WRITE);
    retry:
      item->ov.hEvent = ior;
      if (WriteFileEx(fd, item->single.iov_base, (DWORD)bytes, &item->ov, ior_wocr)) {
        async_started += 1;
      } else {
        r.err = (int)GetLastError();
        switch (r.err) {
        default:
          ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
                " offset %" PRId64 ", err %d",
                "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
                ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
                r.err);
          goto bailout_rc;
        case ERROR_NOT_FOUND:
        case ERROR_USER_MAPPED_FILE:
        case ERROR_LOCK_VIOLATION:
          WARNING("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
                  " offset %" PRId64 ", err %d",
                  "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
                  ((page_t *)item->single.iov_base)->pgno, bytes,
                  item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err);
          SleepEx(0, true);
          goto retry;
        case ERROR_INVALID_USER_BUFFER:
        case ERROR_NOT_ENOUGH_MEMORY:
          if (SleepEx(0, true) == WAIT_IO_COMPLETION)
            goto retry;
          goto bailout_rc;
        case ERROR_IO_PENDING:
          async_started += 1;
        }
      }
    } else {
      ASSERT(bytes < MAX_WRITE);
      DWORD written = 0;
      if (!WriteFile(fd, item->single.iov_base, (DWORD)bytes, &written, &item->ov)) {
        r.err = (int)GetLastError();
        ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
              " offset %" PRId64 ", err %d",
              "WriteFile", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
              ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
              r.err);
        goto bailout_rc;
      } else if (unlikely(written != bytes)) {
        r.err = ERROR_WRITE_FAULT;
        goto bailout_rc;
      }
    }
    item = ior_next(item, i);
  }

  ASSERT(ior->async_waiting > ior->async_completed && ior->async_waiting == INT_MAX);
  ior->async_waiting = async_started;
  if (async_started > ior->async_completed && end_wait_for == wait_for) {
    ASSERT(wait_for > ior->event_pool + ior->event_stack);
    *--wait_for = ior->async_done;
  }

  const size_t pending_count = end_wait_for - wait_for;
  if (pending_count) {
    /* Ждем до MAXIMUM_WAIT_OBJECTS (64) последних хендлов, а после избирательно
     * ждем посредством GetOverlappedResult(), если какие-то более ранние
     * элементы еще не завершены. В целом, так получается меньше системных
     * вызовов, т.е. меньше накладных расходов. Однако, не факт что эта экономия
     * не будет перекрыта неэффективностью реализации
     * WaitForMultipleObjectsEx(), но тогда это проблемы на стороне M$. */
    DWORD madness;
    do
      madness = WaitForMultipleObjectsEx(
          (pending_count < MAXIMUM_WAIT_OBJECTS) ? (DWORD)pending_count : MAXIMUM_WAIT_OBJECTS, wait_for, true,
          /* сутки */ 86400000ul, true);
    while (madness == WAIT_IO_COMPLETION);
    STATIC_ASSERT(WAIT_OBJECT_0 == 0);
    if (/* madness >= WAIT_OBJECT_0 && */
        madness < WAIT_OBJECT_0 + MAXIMUM_WAIT_OBJECTS)
      r.err = MDBX_SUCCESS;
    else if (madness >= WAIT_ABANDONED_0 && madness < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) {
      r.err = ERROR_ABANDONED_WAIT_0;
      goto bailout_rc;
    } else if (madness == WAIT_TIMEOUT) {
      r.err = WAIT_TIMEOUT;
      goto bailout_rc;
    } else {
      r.err = /* madness == WAIT_FAILED */ MDBX_PROBLEM;
      goto bailout_rc;
    }

    ASSERT(ior->async_waiting == ior->async_completed);
    for (ior_item_t *item = ior->pool; item <= ior->last;) {
      size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag;
      void *data = item->single.iov_base;
      if (bytes & ior_WriteFile_flag) {
        data = Ptr64ToPtr(item->sgv[0].Buffer);
        bytes = ior->pagesize;
        /* Zap: Reading invalid data from 'item->sgv' */
        MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
        while (item->sgv[i].Buffer) {
          bytes += ior->pagesize;
          ++i;
        }
        if (!HasOverlappedIoCompleted(&item->ov)) {
          DWORD written = 0;
          if (unlikely(!GetOverlappedResult(fd, &item->ov, &written, true))) {
            ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu,"
                  " offset %" PRId64 ", err %d",
                  "GetOverlappedResult", __Wpedantic_format_voidptr(item), item - ior->pool, data,
                  ((page_t *)data)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
                  (int)GetLastError());
            goto bailout_geterr;
          }
          ASSERT(MDBX_SUCCESS == item->ov.Internal);
          ASSERT(written == item->ov.InternalHigh);
        }
      } else {
        ASSERT(HasOverlappedIoCompleted(&item->ov));
      }
      ASSERT(item->ov.Internal != ERROR_IO_PENDING);
      if (unlikely(item->ov.Internal != MDBX_SUCCESS)) {
        DWORD written = 0;
        r.err = (int)item->ov.Internal;
        if ((r.err & 0x80000000) && GetOverlappedResult(nullptr, &item->ov, &written, true))
          r.err = (int)GetLastError();
        ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu,"
              " offset %" PRId64 ", err %d",
              "Result", __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno, bytes,
              item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), (int)GetLastError());
        goto bailout_rc;
      }
      if (unlikely(item->ov.InternalHigh != bytes)) {
        r.err = ERROR_WRITE_FAULT;
        goto bailout_rc;
      }
      item = ior_next(item, i);
    }
    ASSERT(ior->async_waiting == ior->async_completed);
  } else {
    ASSERT(r.err == MDBX_SUCCESS);
  }
  ASSERT(ior->async_waiting == ior->async_completed);

#else
  STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
  for (ior_item_t *item = ior->pool; item <= ior->last;) {
#if MDBX_HAVE_PWRITEV
    ASSERT(item->sgvcnt > 0);
    if (item->sgvcnt == 1)
      r.err = osal_pwrite(fd, item->sgv[0].iov_base, item->sgv[0].iov_len, item->offset);
    else
      r.err = osal_pwritev(fd, item->sgv, item->sgvcnt, item->offset);

    // TODO: io_uring_prep_write(sqe, fd, ...);

    item = ior_next(item, item->sgvcnt);
#else
    r.err = osal_pwrite(fd, item->single.iov_base, item->single.iov_len, item->offset);
    item = ior_next(item, 1);
#endif
    r.wops += 1;
    if (unlikely(r.err != MDBX_SUCCESS))
      break;
  }

  // TODO: io_uring_submit(&ring)
  // TODO: err = io_uring_wait_cqe(&ring, &cqe);
  // TODO: io_uring_cqe_seen(&ring, cqe);

#endif /* !Windows */
  return r;
}

void osal_ioring_reset(osal_ioring_t *ior) {
#if defined(_WIN32) || defined(_WIN64)
  if (ior->last) {
    for (ior_item_t *item = ior->pool; item <= ior->last;) {
      if (!HasOverlappedIoCompleted(&item->ov)) {
        ASSERT(ior->overlapped_fd);
        CancelIoEx(ior->overlapped_fd, &item->ov);
      }
      if (item->ov.hEvent && item->ov.hEvent != ior)
        ior_put_event(ior, item->ov.hEvent);
      size_t i = 1;
      if ((item->single.iov_len & ior_WriteFile_flag) == 0) {
        /* Zap: Reading invalid data from 'item->sgv' */
        MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
        while (item->sgv[i].Buffer)
          ++i;
      }
      item = ior_next(item, i);
    }
  }
  ior->async_waiting = INT_MAX;
  ior->async_completed = 0;
  ResetEvent(ior->async_done);
#endif /* !Windows */
  ior->slots_left = ior->allocated;
  ior->last = nullptr;
}

static void ior_cleanup(osal_ioring_t *ior, const size_t since) {
  osal_ioring_reset(ior);
#if defined(_WIN32) || defined(_WIN64)
  for (size_t i = since; i < ior->event_stack; ++i) {
    /* Zap: Using uninitialized memory '**ior.event_pool' */
    MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001);
    CloseHandle(ior->event_pool[i]);
  }
  ior->event_stack = 0;
#else
  (void)since;
#endif /* Windows */
}

int osal_ioring_resize(osal_ioring_t *ior, size_t items) {
  ASSERT(items > 0 && items < INT_MAX / sizeof(ior_item_t));
#if defined(_WIN32) || defined(_WIN64)
  if (ior->state & IOR_STATE_LOCKED)
    return MDBX_SUCCESS;
  const bool useSetFileIoOverlappedRange = ior->overlapped_fd && imports.SetFileIoOverlappedRange && items > 42;
  const size_t ceiling =
      useSetFileIoOverlappedRange ? ((items < 65536 / 2 / sizeof(ior_item_t)) ? 65536 : 65536 * 4) : 1024;
  const size_t bytes = ceil_powerof2(sizeof(ior_item_t) * items, ceiling);
  items = bytes / sizeof(ior_item_t);
#endif /* Windows */

  if (items != ior->allocated) {
    ASSERT(items >= osal_ioring_used(ior));
    if (items < ior->allocated)
      ior_cleanup(ior, items);
#if defined(_WIN32) || defined(_WIN64)
    void *ptr = osal_realloc(ior->event_pool, (items + /* extra for waiting the async_done */ 1) * sizeof(HANDLE));
    if (unlikely(!ptr))
      return MDBX_ENOMEM;
    ior->event_pool = ptr;

    int err = osal_memalign_alloc(ceiling, bytes, &ptr);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if (ior->pool) {
      memcpy(ptr, ior->pool, ior->allocated * sizeof(ior_item_t));
      osal_memalign_free(ior->pool);
    }
#else
    void *ptr = osal_realloc(ior->pool, sizeof(ior_item_t) * items);
    if (unlikely(!ptr))
      return MDBX_ENOMEM;
#endif
    ior->pool = ptr;

    if (items > ior->allocated)
      memset(ior->pool + ior->allocated, 0, sizeof(ior_item_t) * (items - ior->allocated));
    ior->allocated = (unsigned)items;
    ior->boundary = ptr_disp(ior->pool, ior->allocated);
#if defined(_WIN32) || defined(_WIN64)
    if (useSetFileIoOverlappedRange) {
      if (imports.SetFileIoOverlappedRange(ior->overlapped_fd, ptr, (ULONG)bytes))
        ior->state += IOR_STATE_LOCKED;
      else
        return GetLastError();
    }
#endif /* Windows */
  }
  return MDBX_SUCCESS;
}

void osal_ioring_destroy(osal_ioring_t *ior) {
  if (ior->allocated)
    ior_cleanup(ior, 0);
#if defined(_WIN32) || defined(_WIN64)
  osal_memalign_free(ior->pool);
  osal_free(ior->event_pool);
  CloseHandle(ior->async_done);
  if (ior->overlapped_fd)
    CloseHandle(ior->overlapped_fd);
#else
  osal_free(ior->pool);
#endif
  memset(ior, 0, sizeof(osal_ioring_t));
}

/*----------------------------------------------------------------------------*/

int osal_removefile(const pathchar_t *pathname) {
#if defined(_WIN32) || defined(_WIN64)
  return DeleteFileW(pathname) ? MDBX_SUCCESS : (int)GetLastError();
#else
  return unlink(pathname) ? errno : MDBX_SUCCESS;
#endif
}

#if !(defined(_WIN32) || defined(_WIN64))
static bool is_valid_fd(int fd) { return !(isatty(fd) < 0 && errno == EBADF); }
#endif /*! Windows */

int osal_removedirectory(const pathchar_t *pathname) {
#if defined(_WIN32) || defined(_WIN64)
  return RemoveDirectoryW(pathname) ? MDBX_SUCCESS : (int)GetLastError();
#else
  return rmdir(pathname) ? errno : MDBX_SUCCESS;
#endif
}

int osal_fileexists(const pathchar_t *pathname) {
#if defined(_WIN32) || defined(_WIN64)
  if (GetFileAttributesW(pathname) != INVALID_FILE_ATTRIBUTES)
    return MDBX_RESULT_TRUE;
  int err = GetLastError();
  return (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) ? MDBX_RESULT_FALSE : err;
#else
  if (access(pathname, F_OK) == 0)
    return MDBX_RESULT_TRUE;
  int err = errno;
  return (err == ENOENT || err == ENOTDIR) ? MDBX_RESULT_FALSE : err;
#endif
}

pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len) {
  const pathchar_t *ext = nullptr;
  for (size_t i = 0; i < len && pathname[i]; i++)
    if (pathname[i] == '.')
      ext = pathname + i;
    else if (osal_isdirsep(pathname[i]))
      ext = nullptr;
  return (pathchar_t *)ext;
}

bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len) {
#if defined(_WIN32) || defined(_WIN64)
  for (size_t i = 0; i < len; ++i) {
    pathchar_t a = l[i];
    pathchar_t b = r[i];
    a = (a == '\\') ? '/' : a;
    b = (b == '\\') ? '/' : b;
    if (a != b)
      return false;
  }
  return true;
#else
  return memcmp(l, r, len * sizeof(pathchar_t)) == 0;
#endif
}

#if !(defined(_WIN32) || defined(_WIN64))
static const char dev_null[] = "/dev/null";
#endif /* !Windows */

int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, const pathchar_t *pathname,
                  mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits) {
  *fd = INVALID_HANDLE_VALUE;

#if defined(_WIN32) || defined(_WIN64)
  DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING;
  DWORD FlagsAndAttributes = FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED;
  DWORD DesiredAccess = FILE_READ_ATTRIBUTES;
  DWORD ShareMode = (env->flags & MDBX_EXCLUSIVE) ? 0 : (FILE_SHARE_READ | FILE_SHARE_WRITE);

  switch (purpose) {
  default:
    return ERROR_INVALID_PARAMETER;
  case MDBX_OPEN_LCK:
    CreationDisposition = OPEN_ALWAYS;
    DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
    FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_OVERLAPPED;
    break;
  case MDBX_OPEN_DXB_READ:
    CreationDisposition = OPEN_EXISTING;
    DesiredAccess |= GENERIC_READ;
    ShareMode |= FILE_SHARE_READ;
    break;
  case MDBX_OPEN_DXB_LAZY:
    DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
    break;
  case MDBX_OPEN_DXB_OVERLAPPED_DIRECT:
    FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING;
    __fallthrough /* fall through */;
  case MDBX_OPEN_DXB_OVERLAPPED:
    FlagsAndAttributes |= FILE_FLAG_OVERLAPPED;
    __fallthrough /* fall through */;
  case MDBX_OPEN_DXB_DSYNC:
    CreationDisposition = OPEN_EXISTING;
    DesiredAccess |= GENERIC_WRITE | GENERIC_READ;
    FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH;
    break;
  case MDBX_OPEN_DELETE:
    CreationDisposition = OPEN_EXISTING;
    ShareMode |= FILE_SHARE_DELETE;
    DesiredAccess = FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE;
    break;
  case MDBX_OPEN_COPY_EXCL:
    CreationDisposition = CREATE_NEW;
    __fallthrough /* fall through */;
  case MDBX_OPEN_COPY_OVERWRITE:
    ShareMode = 0;
    DesiredAccess |= GENERIC_WRITE;
    if (env->ps >= globals.sys_pagesize)
      FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING;
    break;
  }

  *fd = CreateFileW(pathname, DesiredAccess, ShareMode, nullptr, CreationDisposition, FlagsAndAttributes, nullptr);
  if (*fd == INVALID_HANDLE_VALUE) {
    int err = (int)GetLastError();
    if (err == ERROR_ACCESS_DENIED && purpose == MDBX_OPEN_LCK) {
      if (GetFileAttributesW(pathname) == INVALID_FILE_ATTRIBUTES && GetLastError() == ERROR_FILE_NOT_FOUND)
        err = ERROR_FILE_NOT_FOUND;
    }
    return err;
  }

  BY_HANDLE_FILE_INFORMATION info;
  if (!GetFileInformationByHandle(*fd, &info)) {
    int err = (int)GetLastError();
    CloseHandle(*fd);
    *fd = INVALID_HANDLE_VALUE;
    return err;
  }
  const DWORD AttributesDiff =
      (info.dwFileAttributes ^ FlagsAndAttributes) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |
                                                      FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED);
  if (AttributesDiff)
    (void)SetFileAttributesW(pathname, info.dwFileAttributes ^ AttributesDiff);

#else
  int flags = unix_mode_bits ? O_CREAT : 0;
  switch (purpose) {
  default:
    return EINVAL;
  case MDBX_OPEN_LCK:
    flags |= O_RDWR;
    break;
  case MDBX_OPEN_DXB_READ:
    flags = O_RDONLY;
    break;
  case MDBX_OPEN_DXB_LAZY:
    flags |= O_RDWR;
    break;
  case MDBX_OPEN_DXB_DSYNC:
    flags |= O_WRONLY;
#if defined(O_DSYNC)
    flags |= O_DSYNC;
#elif defined(O_SYNC)
    flags |= O_SYNC;
#elif defined(O_FSYNC)
    flags |= O_FSYNC;
#endif
    break;
  case MDBX_OPEN_DELETE:
    flags = O_RDWR;
    break;
  case MDBX_OPEN_COPY_EXCL:
    flags |= O_EXCL;
    __fallthrough /* fall through */;
  case MDBX_OPEN_COPY_OVERWRITE:
    flags |= O_WRONLY;
  }

  const bool direct_nocache_for_copy = env->ps >= globals.sys_pagesize && purpose >= MDBX_OPEN_COPY_EXCL;
  if (direct_nocache_for_copy) {
#if defined(O_DIRECT)
    flags |= O_DIRECT;
#endif /* O_DIRECT */
#if defined(O_NOCACHE)
    flags |= O_NOCACHE;
#endif /* O_NOCACHE */
  }

#ifdef O_CLOEXEC
  flags |= O_CLOEXEC;
#endif /* O_CLOEXEC */

  /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */
#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
  int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1;
  if (!is_valid_fd(STDIN_FILENO)) {
    WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN", STDIN_FILENO, dev_null);
    stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY);
  }
  if (!is_valid_fd(STDOUT_FILENO)) {
    WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "OUT", STDOUT_FILENO, dev_null);
    stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY);
  }
  if (!is_valid_fd(STDERR_FILENO)) {
    WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "ERR", STDERR_FILENO, dev_null);
    stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY);
  }
#else
#error "Unexpected or unsupported UNIX or POSIX system"
#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */

  *fd = open(pathname, flags, unix_mode_bits);
#if defined(O_DIRECT)
  if (*fd < 0 && (flags & O_DIRECT) && (errno == EINVAL || errno == EAFNOSUPPORT)) {
    flags &= ~(O_DIRECT | O_EXCL);
    *fd = open(pathname, flags, unix_mode_bits);
  }
#endif /* O_DIRECT */

  if (*fd < 0 && errno == EACCES && purpose == MDBX_OPEN_LCK) {
    struct stat unused;
    if (stat(pathname, &unused) == 0 || errno != ENOENT)
      errno = EACCES /* restore errno if file exists */;
  }

  /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */
#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
  if (*fd == STDIN_FILENO) {
    WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN", STDIN_FILENO);
    ASSERT(stub_fd0 == -1);
    *fd = dup(stub_fd0 = *fd);
  }
  if (*fd == STDOUT_FILENO) {
    WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT", STDOUT_FILENO);
    ASSERT(stub_fd1 == -1);
    *fd = dup(stub_fd1 = *fd);
  }
  if (*fd == STDERR_FILENO) {
    WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR", STDERR_FILENO);
    ASSERT(stub_fd2 == -1);
    *fd = dup(stub_fd2 = *fd);
  }
  const int err = errno;
  if (stub_fd0 != -1)
    close(stub_fd0);
  if (stub_fd1 != -1)
    close(stub_fd1);
  if (stub_fd2 != -1)
    close(stub_fd2);
  if (*fd >= STDIN_FILENO && *fd <= STDERR_FILENO) {
    ERROR("Rejecting the use of a FD in the range "
          "STDIN_FILENO/%d..STDERR_FILENO/%d to prevent database corruption",
          STDIN_FILENO, STDERR_FILENO);
    close(*fd);
    return EBADF;
  }
#else
#error "Unexpected or unsupported UNIX or POSIX system"
#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */

  if (*fd < 0)
    return err;

#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC)
  const int fd_flags = fcntl(*fd, F_GETFD);
  if (fd_flags != -1)
    (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC);
#endif /* FD_CLOEXEC && !O_CLOEXEC */

  if (direct_nocache_for_copy) {
#if defined(F_NOCACHE) && !defined(O_NOCACHE)
    (void)fcntl(*fd, F_NOCACHE, 1);
#endif /* F_NOCACHE */
  }

#endif
  return MDBX_SUCCESS;
}

int osal_closefile(mdbx_filehandle_t fd) {
#if defined(_WIN32) || defined(_WIN64)
  return CloseHandle(fd) ? MDBX_SUCCESS : (int)GetLastError();
#else
  ASSERT(fd > STDERR_FILENO);
  return (close(fd) == 0) ? MDBX_SUCCESS : errno;
#endif
}

int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, uint64_t offset) {
  if (bytes > MAX_WRITE)
    return MDBX_EINVAL;
#if defined(_WIN32) || defined(_WIN64)
  OVERLAPPED ov;
  ov.hEvent = 0;
  ov.Offset = (DWORD)offset;
  ov.OffsetHigh = HIGH_DWORD(offset);

  DWORD read = 0;
  if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) {
    int err = (int)GetLastError();
    if (err != ERROR_IO_PENDING)
      return (err == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : err;
    if (!GetOverlappedResult(fd, &ov, &read, true)) {
      err = (int)GetLastError();
      CancelIo(fd);
      return (err == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : err;
    }
  }
#else
  STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
  intptr_t read = pread(fd, buf, bytes, offset);
  if (read < 0) {
    int rc = errno;
    return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc;
  }
#endif
  return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA;
}

int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t bytes, uint64_t offset) {
  while (true) {
#if defined(_WIN32) || defined(_WIN64)
    OVERLAPPED ov;
    ov.hEvent = 0;
    ov.Offset = (DWORD)offset;
    ov.OffsetHigh = HIGH_DWORD(offset);

    DWORD written;
    if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, &ov)))
      return (int)GetLastError();
    if (likely(bytes == written))
      return MDBX_SUCCESS;
#else
    STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
    const intptr_t written = pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset);
    if (likely(bytes == (size_t)written))
      return MDBX_SUCCESS;
    if (written < 0) {
      const int rc = errno;
      if (rc != EINTR)
        return rc;
      continue;
    }
#endif
    bytes -= written;
    offset += written;
    buf = ptr_disp(buf, written);
  }
}

int osal_write(mdbx_filehandle_t fd, const void *buf, size_t bytes) {
  while (true) {
#if defined(_WIN32) || defined(_WIN64)
    DWORD written;
    if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, nullptr)))
      return (int)GetLastError();
    if (likely(bytes == written))
      return MDBX_SUCCESS;
#else
    STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
    const intptr_t written = write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE);
    if (likely(bytes == (size_t)written))
      return MDBX_SUCCESS;
    if (written < 0) {
      const int rc = errno;
      if (rc != EINTR)
        return rc;
      continue;
    }
#endif
    bytes -= written;
    buf = ptr_disp(buf, written);
  }
}

int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset) {
  size_t expected = 0;
  for (size_t i = 0; i < sgvcnt; ++i)
    expected += iov[i].iov_len;
#if !MDBX_HAVE_PWRITEV
  size_t written = 0;
  for (size_t i = 0; i < sgvcnt; ++i) {
    int rc = osal_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    written += iov[i].iov_len;
    offset += iov[i].iov_len;
  }
  return (expected == written) ? MDBX_SUCCESS : MDBX_EIO /* ERROR_WRITE_FAULT */;
#else
  int rc;
  intptr_t written;
  do {
    STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
    written = pwritev(fd, iov, sgvcnt, offset);
    if (likely(expected == (size_t)written))
      return MDBX_SUCCESS;
    rc = errno;
  } while (rc == EINTR);
  return (written < 0) ? rc : MDBX_EIO /* Use which error code? */;
#endif
}

int osal_fsync(mdbx_filehandle_t fd, enum osal_syncmode_bits mode_bits) {
#if defined(_WIN32) || defined(_WIN64)
  if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd))
    return (int)GetLastError();
  return MDBX_SUCCESS;
#else

#if defined(__APPLE__) && MDBX_APPLE_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY
  if (mode_bits & MDBX_SYNC_IODQ)
    return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno;
#endif /* MacOS */

  /* LY: This approach is always safe and without appreciable performance
   * degradation, even on a kernel with fdatasync's bug.
   *
   * For more info about of a corresponding fdatasync() bug
   * see http://www.spinics.net/lists/linux-ext4/msg33714.html */
  while (1) {
    switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) {
    case MDBX_SYNC_NONE:
    case MDBX_SYNC_KICK:
      return MDBX_SUCCESS /* nothing to do */;
#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
    case MDBX_SYNC_DATA:
      if (likely(fdatasync(fd) == 0))
        return MDBX_SUCCESS;
      break /* error */;
#if defined(__linux__) || defined(__gnu_linux__)
    case MDBX_SYNC_SIZE:
      ASSERT(globals.linux_kernel_version >= 0x03060000);
      return MDBX_SUCCESS;
#endif /* Linux */
#endif /* _POSIX_SYNCHRONIZED_IO > 0 */
    default:
      if (likely(fsync(fd) == 0))
        return MDBX_SUCCESS;
    }

    int rc = errno;
    if (rc != EINTR)
      return rc;
  }
#endif
}

int osal_filesize(mdbx_filehandle_t fd, uint64_t *length) {
#if defined(_WIN32) || defined(_WIN64)
  BY_HANDLE_FILE_INFORMATION info;
  if (!GetFileInformationByHandle(fd, &info))
    return (int)GetLastError();
  *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32;
#else
  struct stat st;

  STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
  if (fstat(fd, &st))
    return errno;

  *length = st.st_size;
#endif
  return MDBX_SUCCESS;
}

int osal_is_pipe(mdbx_filehandle_t fd) {
#if defined(_WIN32) || defined(_WIN64)
  switch (GetFileType(fd)) {
  case FILE_TYPE_DISK:
    return MDBX_RESULT_FALSE;
  case FILE_TYPE_CHAR:
  case FILE_TYPE_PIPE:
    return MDBX_RESULT_TRUE;
  default:
    return (int)GetLastError();
  }
#else
  struct stat info;
  if (fstat(fd, &info))
    return errno;
  switch (info.st_mode & S_IFMT) {
  case S_IFBLK:
  case S_IFREG:
    return MDBX_RESULT_FALSE;
  case S_IFCHR:
  case S_IFIFO:
  case S_IFSOCK:
    return MDBX_RESULT_TRUE;
  case S_IFDIR:
  case S_IFLNK:
  default:
    return MDBX_INCOMPATIBLE;
  }
#endif
}

int osal_fsetsize(mdbx_filehandle_t fd, const uint64_t length) {
#if defined(_WIN32) || defined(_WIN64)
  if (imports.SetFileInformationByHandle) {
    FILE_END_OF_FILE_INFO EndOfFileInfo;
    EndOfFileInfo.EndOfFile.QuadPart = length;
    return imports.SetFileInformationByHandle(fd, FileEndOfFileInfo, &EndOfFileInfo, sizeof(FILE_END_OF_FILE_INFO))
               ? MDBX_SUCCESS
               : (int)GetLastError();
  } else {
    LARGE_INTEGER li;
    li.QuadPart = length;
    return (SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) && SetEndOfFile(fd)) ? MDBX_SUCCESS : (int)GetLastError();
  }
#else
  STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");

#if MDBX_USE_FALLOCATE
  struct stat info;
  if (unlikely(fstat(fd, &info)))
    return errno;

  const uint64_t allocated = UINT64_C(512) * info.st_blocks;
  if (length > allocated) {
#if defined(__APPLE__)
    fstore_t store = {
        .fst_flags = F_ALLOCATECONTIG, .fst_posmode = F_PEOFPOSMODE, .fst_offset = 0, .fst_length = length};
    int err = MDBX_SUCCESS;
    if (fcntl(fd, F_PREALLOCATE, &store)) {
      /* TODO: implement step-by-step allocation in chunks of 16384, 8192, 4094, 2048, 1024 Kb */
      store.fst_flags = F_ALLOCATEALL;
      if (fcntl(fd, F_PREALLOCATE, &store))
        err = errno;
    }
#elif defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L
    const int err = posix_fallocate(fd, 0, length);
    if (!err && length > (uint64_t)info.st_size)
      info.st_size = length /* posix_fallocate() extends the file */;
#else
    const int err = fallocate(fd, 0, 0, length) ? errno : MDBX_SUCCESS;
    if (!err && length > (uint64_t)info.st_size)
      info.st_size = length /* fallocate() extends the file */;
#endif
    if (unlikely(err) && ignore_enosys_and_eremote(err) != MDBX_RESULT_TRUE) {
      /* Workaround for testing: ignore ENOSPC for TMPFS/RAMFS.
       * This is insignificant for production, but it helps in some tests using /dev/shm inside docker/containers. */
      if (err != ENOSPC || osal_check_fs_incore(fd) != MDBX_RESULT_TRUE)
        return err;
    }
  }

  if (length == (uint64_t)info.st_size)
    return MDBX_SUCCESS;
#endif

#if defined(__linux__) || defined(__gnu_linux__)
  if (globals.linux_kernel_version < 0x05110000 && globals.linux_kernel_version >= 0x050a0000) {
    struct statfs statfs_info;
    if (fstatfs(fd, &statfs_info))
      return errno;
    if (statfs_info.f_type == 0xEF53 /* EXT4_SUPER_MAGIC */ && unlikely(fdatasync(fd)))
      return errno;
  }
#endif /* Linux */

  return unlikely(ftruncate(fd, length)) ? errno : MDBX_SUCCESS;

#endif /* !Windows */
}

int osal_fseek(mdbx_filehandle_t fd, uint64_t pos) {
#if defined(_WIN32) || defined(_WIN64)
  LARGE_INTEGER li;
  li.QuadPart = pos;
  return SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) ? MDBX_SUCCESS : (int)GetLastError();
#else
  STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
  return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS;
#endif
}

/*----------------------------------------------------------------------------*/

int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), void *arg) {
#if defined(_WIN32) || defined(_WIN64)
  *thread = CreateThread(nullptr, 0, start_routine, arg, 0, nullptr);
  return *thread ? MDBX_SUCCESS : (int)GetLastError();
#else
  return pthread_create(thread, nullptr, start_routine, arg);
#endif
}

int osal_thread_join(osal_thread_t thread) {
#if defined(_WIN32) || defined(_WIN64)
  DWORD code = WaitForSingleObject(thread, INFINITE);
  return osal_waitstatus2errcode(code);
#else
  void *unused_retval = &unused_retval;
  return pthread_join(thread, &unused_retval);
#endif
}

/*----------------------------------------------------------------------------*/

int osal_msync(const osal_mmap_t *map, size_t length, enum osal_syncmode_bits mode_bits) {
  if (!MDBX_MMAP_NEEDS_JOLT && mode_bits == MDBX_SYNC_NONE)
    return MDBX_SUCCESS;

#if defined(_WIN32) || defined(_WIN64)
  if (!FlushViewOfFile(map->base, length))
    return (int)GetLastError();
  if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(map->fd))
    return (int)GetLastError();
#else
#if defined(__linux__) || defined(__gnu_linux__)
  /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly
   * tracks dirty pages and flushes ones as necessary. */
  //
  // However, this behavior may be changed in custom kernels,
  // so just leave such optimization to the libc discretion.
  // NOTE: The MDBX_MMAP_NEEDS_JOLT must be defined to 1 for such cases.
  //
  // ASSERT(mdbx.linux_kernel_version > 0x02061300);
  // if (mode_bits <= MDBX_SYNC_KICK)
  //   return MDBX_SUCCESS;
#endif /* Linux */
  if (msync(map->base, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC))
    return errno;
  if ((mode_bits & MDBX_SYNC_SIZE) && fsync(map->fd))
    return errno;
#endif
  return MDBX_SUCCESS;
}

int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err) {
#if defined(_WIN32) || defined(_WIN64)
  (void)pathname;
  (void)err;
  if (!imports.GetVolumeInformationByHandleW)
    return MDBX_ENOSYS;
  DWORD unused, flags;
  if (!imports.GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, &flags, nullptr, 0))
    return (int)GetLastError();
  if ((flags & FILE_READ_ONLY_VOLUME) == 0)
    return MDBX_EACCESS;
#else
  struct statvfs info;
  if (err != MDBX_ENOFILE) {
    if (statvfs(pathname, &info) == 0)
      return (info.f_flag & ST_RDONLY) ? MDBX_SUCCESS : err;
    if (errno != MDBX_ENOFILE)
      return errno;
  }
  if (fstatvfs(handle, &info))
    return errno;
  if ((info.f_flag & ST_RDONLY) == 0)
    return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err;
#endif /* !Windows */
  return MDBX_SUCCESS;
}

int osal_check_fs_incore(mdbx_filehandle_t handle) {
#if defined(_WIN32) || defined(_WIN64)
  (void)handle;
#else
  struct statfs statfs_info;
  if (fstatfs(handle, &statfs_info))
    return errno;

#if defined(__OpenBSD__)
  const unsigned type = 0;
#else
  const unsigned type = statfs_info.f_type;
#endif
  switch (type) {
  case 0x28cd3d45 /* CRAMFS_MAGIC */:
  case 0x858458f6 /* RAMFS_MAGIC */:
  case 0x01021994 /* TMPFS_MAGIC */:
  case 0x73717368 /* SQUASHFS_MAGIC */:
  case 0x7275 /* ROMFS_MAGIC */:
    return MDBX_RESULT_TRUE;
  }

#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) ||    \
    defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) ||                        \
    defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
  const char *const name = statfs_info.f_fstypename;
  const size_t name_len = sizeof(statfs_info.f_fstypename);
#else
  const char *const name = "";
  const size_t name_len = 0;
#endif
  if (name_len) {
    if (strncasecmp("tmpfs", name, 6) == 0 || strncasecmp("mfs", name, 4) == 0 || strncasecmp("ramfs", name, 6) == 0 ||
        strncasecmp("romfs", name, 6) == 0)
      return MDBX_RESULT_TRUE;
  }
#endif /* !Windows */

  return MDBX_RESULT_FALSE;
}

int osal_check_fs_local(mdbx_filehandle_t handle, int flags) {
#if defined(_WIN32) || defined(_WIN64)
  if (globals.running_under_Wine && !(flags & MDBX_EXCLUSIVE))
    return ERROR_NOT_CAPABLE /* workaround for Wine */;

  if (GetFileType(handle) != FILE_TYPE_DISK)
    return ERROR_FILE_OFFLINE;

  if (imports.GetFileInformationByHandleEx) {
    FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo;
    if (imports.GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, &RemoteProtocolInfo,
                                             sizeof(RemoteProtocolInfo))) {
      if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && !(flags & MDBX_RDONLY))
        return ERROR_FILE_OFFLINE;
      if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && !(flags & MDBX_EXCLUSIVE))
        return MDBX_EREMOTE;
    }
  }

  if (imports.NtFsControlFile) {
    NTSTATUS rc;
    struct {
      WOF_EXTERNAL_INFO wof_info;
      union {
        WIM_PROVIDER_EXTERNAL_INFO wim_info;
        FILE_PROVIDER_EXTERNAL_INFO_V1 file_info;
      };
      size_t reserved_for_microsoft_madness[42];
    } GetExternalBacking_OutputBuffer;
    IO_STATUS_BLOCK StatusBlock;
    rc = imports.NtFsControlFile(handle, nullptr, nullptr, nullptr, &StatusBlock, FSCTL_GET_EXTERNAL_BACKING, nullptr,
                                 0, &GetExternalBacking_OutputBuffer, sizeof(GetExternalBacking_OutputBuffer));
    if (NT_SUCCESS(rc)) {
      if (!(flags & MDBX_EXCLUSIVE))
        return MDBX_EREMOTE;
    } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && rc != STATUS_INVALID_DEVICE_REQUEST &&
               rc != STATUS_NOT_SUPPORTED)
      return osal_ntstatus2errcode(rc);
  }

  if (imports.GetVolumeInformationByHandleW && imports.GetFinalPathNameByHandleW) {
    WCHAR *PathBuffer = osal_malloc(sizeof(WCHAR) * INT16_MAX);
    if (!PathBuffer)
      return MDBX_ENOMEM;

    int rc = MDBX_SUCCESS;
    DWORD VolumeSerialNumber, FileSystemFlags;
    if (!imports.GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, &VolumeSerialNumber, nullptr,
                                               &FileSystemFlags, nullptr, 0)) {
      rc = (int)GetLastError();
      goto bailout;
    }

    if ((flags & MDBX_RDONLY) == 0) {
      if (FileSystemFlags & (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | FILE_VOLUME_IS_COMPRESSED)) {
        rc = MDBX_EREMOTE;
        goto bailout;
      }
    }

    if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_NT)) {
      if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) {
        if (!(flags & MDBX_EXCLUSIVE)) {
          rc = MDBX_EREMOTE;
          goto bailout;
        }
      }
    }

    if (F_ISSET(flags, MDBX_RDONLY | MDBX_EXCLUSIVE) && (FileSystemFlags & FILE_READ_ONLY_VOLUME)) {
      /* without-LCK (exclusive readonly) mode for DB on a read-only volume */
      goto bailout;
    }

    if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS)) {
      UINT DriveType = GetDriveTypeW(PathBuffer);
      if (DriveType == DRIVE_NO_ROOT_DIR && _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 &&
          _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) {
        PathBuffer[7] = 0;
        DriveType = GetDriveTypeW(PathBuffer + 4);
      }
      switch (DriveType) {
      case DRIVE_CDROM:
        if (flags & MDBX_RDONLY)
          break;
      // fall through
      case DRIVE_UNKNOWN:
      case DRIVE_NO_ROOT_DIR:
      case DRIVE_REMOTE:
      default:
        if (!(flags & MDBX_EXCLUSIVE))
          rc = MDBX_EREMOTE;
      // fall through
      case DRIVE_REMOVABLE:
      case DRIVE_FIXED:
      case DRIVE_RAMDISK:
        break;
      }
    }

  bailout:
    osal_free(PathBuffer);
    return rc;
  }

#else

  struct statvfs statvfs_info;
  if (fstatvfs(handle, &statvfs_info))
    return errno;
#if defined(ST_LOCAL) || defined(ST_EXPORTED)
  const unsigned long st_flags = statvfs_info.f_flag;
#endif /* ST_LOCAL || ST_EXPORTED */

#if defined(__NetBSD__)
  const unsigned type = 0;
  const char *const name = statvfs_info.f_fstypename;
  const size_t name_len = VFS_NAMELEN;
#elif defined(_AIX) || defined(__OS400__)
  const char *const name = statvfs_info.f_basetype;
  const size_t name_len = sizeof(statvfs_info.f_basetype);
  struct stat st;
  if (fstat(handle, &st))
    return errno;
  const unsigned type = st.st_vfstype;
  if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE))
    return MDBX_EREMOTE;
#elif defined(FSTYPSZ) || defined(_FSTYPSZ)
  const unsigned type = 0;
  const char *const name = statvfs_info.f_basetype;
  const size_t name_len = sizeof(statvfs_info.f_basetype);
#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ)
  const unsigned type = 0;
  struct stat st;
  if (fstat(handle, &st))
    return errno;
  const char *const name = st.st_fstype;
  const size_t name_len = strlen(name);
#else
  struct statfs statfs_info;
  if (fstatfs(handle, &statfs_info))
    return errno;
#if defined(__OpenBSD__)
  const unsigned type = 0;
#else
  const unsigned type = statfs_info.f_type;
#endif
#if defined(MNT_LOCAL) || defined(MNT_EXPORTED)
  const unsigned long mnt_flags = statfs_info.f_flags;
#endif /* MNT_LOCAL || MNT_EXPORTED */
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) ||    \
    defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) ||                        \
    defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
  const char *const name = statfs_info.f_fstypename;
  const size_t name_len = sizeof(statfs_info.f_fstypename);
#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21
  const char *const name = "";
  const unsigned name_len = 0;
#else

  const char *name = "";
  unsigned name_len = 0;

  struct stat st;
  if (fstat(handle, &st))
    return errno;

  char pathbuf[PATH_MAX];
  FILE *mounted = nullptr;
#if defined(__linux__) || defined(__gnu_linux__)
  mounted = setmntent("/proc/mounts", "r");
#endif /* Linux */
  if (!mounted)
    mounted = setmntent("/etc/mtab", "r");
  if (mounted) {
    const struct mntent *ent;
#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) ||                                            \
    (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19))
    struct mntent entbuf;
    const bool should_copy = false;
    while (nullptr != (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf))))
#else
    const bool should_copy = true;
    while (nullptr != (ent = getmntent(mounted)))
#endif
    {
      struct stat mnt;
      if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) {
        if (should_copy) {
          name = strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1);
          pathbuf[name_len] = 0;
        } else {
          name = ent->mnt_fsname;
          name_len = strlen(name);
        }
        break;
      }
    }
    endmntent(mounted);
  }
#endif /* !xBSD && !Android/Bionic */
#endif

  if (name_len) {
    if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || strncasecmp("cifs", name, name_len) == 0 ||
         strncasecmp("ncpfs", name, name_len) == 0 || strncasecmp("smbfs", name, name_len) == 0 ||
         strcasecmp("9P" /* WSL2 */, name) == 0 ||
         ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && strncasecmp("fuseblk", name, name_len) != 0)) &&
        !(flags & MDBX_EXCLUSIVE))
      return MDBX_EREMOTE;
    if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || strcasecmp("sshfs", name) == 0)
      return MDBX_EREMOTE;
  }

#ifdef ST_LOCAL
  if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
    return MDBX_EREMOTE;
#elif defined(MNT_LOCAL)
  if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
    return MDBX_EREMOTE;
#endif /* ST/MNT_LOCAL */

#ifdef ST_EXPORTED
  if ((st_flags & ST_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)))
    return MDBX_RESULT_TRUE;
#elif defined(MNT_EXPORTED)
  if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)))
    return MDBX_RESULT_TRUE;
#endif /* ST/MNT_EXPORTED */

  switch (type) {
  case 0xFF534D42 /* CIFS_MAGIC_NUMBER */:
  case 0x6969 /* NFS_SUPER_MAGIC */:
  case 0x564c /* NCP_SUPER_MAGIC */:
  case 0x517B /* SMB_SUPER_MAGIC */:
#if defined(__digital__) || defined(__osf__) || defined(__osf)
  case 0x0E /* Tru64 NFS */:
#endif
#ifdef ST_FST_NFS
  case ST_FST_NFS:
#endif
    if ((flags & MDBX_EXCLUSIVE) == 0)
      return MDBX_EREMOTE;
  case 0:
  default:
    break;
  }
#endif /* Unix */

  return MDBX_SUCCESS;
}

static int check_mmap_limit(const size_t limit) {
  const bool should_check =
#if defined(__SANITIZE_ADDRESS__)
      true;
#else
      RUNNING_ON_VALGRIND;
#endif /* __SANITIZE_ADDRESS__ */

  if (should_check) {
    intptr_t pagesize, total_ram_pages, avail_ram_pages;
    int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    const int log2page = log2n_powerof2(pagesize);
    if ((limit >> (log2page + 7)) > (size_t)total_ram_pages || (limit >> (log2page + 6)) > (size_t)avail_ram_pages) {
      ERROR("%s (%zu pages) is too large for available (%zu pages) or total "
            "(%zu pages) system RAM",
            "database upper size limit", limit >> log2page, avail_ram_pages, total_ram_pages);
      return MDBX_TOO_LARGE;
    }
  }

  return MDBX_SUCCESS;
}

int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options,
              const pathchar_t *pathname4logging) {
  ASSERT(size <= limit);
  map->limit = 0;
  map->current = 0;
  map->base = nullptr;
  map->filesize = 0;
#if defined(_WIN32) || defined(_WIN64)
  map->section = nullptr;
#endif /* Windows */

  int err = osal_check_fs_local(map->fd, flags);
  if (unlikely(err != MDBX_SUCCESS)) {
#if defined(_WIN32) || defined(_WIN64)
    if (globals.running_under_Wine)
      NOTICE("%s", "Please use native Linux application or WSL at least, instead of trouble-full Wine!");
#endif /* Windows */
    switch (err) {
    case MDBX_RESULT_TRUE:
#if MDBX_ENABLE_NON_READONLY_EXPORT
      WARNING("%" MDBX_PRIsPATH " is exported via NFS, avoid using the file on a remote side!", pathname4logging);
      break;
#else
      ERROR("%" MDBX_PRIsPATH " is exported via NFS", pathname4logging);
      return MDBX_EREMOTE;
#endif /* MDBX_PROHIBIT_NON_READONLY_EXPORT */
    case MDBX_EREMOTE:
      ERROR("%" MDBX_PRIsPATH " is on a remote file system, the %s is required", pathname4logging, "MDBX_EXCLUSIVE");
      __fallthrough /* fall through */;
    default:
      return err;
    }
  }

  err = check_mmap_limit(limit);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_SETLENGTH) != 0) {
    err = osal_fsetsize(map->fd, size);
    VERBOSE("osal_fsetsize %zu, err %d", size, err);
    if (err != MDBX_SUCCESS)
      return err;
    map->filesize = size;
#if !(defined(_WIN32) || defined(_WIN64))
    map->current = size;
#endif /* !Windows */
  } else {
    err = osal_filesize(map->fd, &map->filesize);
    VERBOSE("filesize %" PRIu64 ", err %d", map->filesize, err);
    if (err != MDBX_SUCCESS)
      return err;
#if defined(_WIN32) || defined(_WIN64)
    if (map->filesize < size) {
      WARNING("file size (%zu) less than requested for mapping (%zu)", (size_t)map->filesize, size);
      size = (size_t)map->filesize;
    }
#else
    map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
#endif /* !Windows */
  }

#if defined(_WIN32) || defined(_WIN64)
  LARGE_INTEGER SectionSize;
  SectionSize.QuadPart = size;
  err = NtCreateSection(&map->section,
                        /* DesiredAccess */
                        (flags & MDBX_WRITEMAP)
                            ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE
                            : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
                        /* ObjectAttributes */ nullptr,
                        /* MaximumSize (InitialSize) */ &SectionSize,
                        /* SectionPageProtection */
                        (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
                        /* AllocationAttributes */ SEC_RESERVE, map->fd);
  if (!NT_SUCCESS(err))
    return osal_ntstatus2errcode(err);

  SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 : globals.running_under_Wine ? size : limit;
  err = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base,
                           /* ZeroBits */ 0,
                           /* CommitSize */ 0,
                           /* SectionOffset */ nullptr, &ViewSize,
                           /* InheritDisposition */ ViewUnmap,
                           /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
                           /* Win32Protect */
                           (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
  if (!NT_SUCCESS(err)) {
    NtClose(map->section);
    map->section = 0;
    map->base = nullptr;
    return osal_ntstatus2errcode(err);
  }
  ASSERT(map->base != MAP_FAILED);

  map->current = (size_t)SectionSize.QuadPart;
  map->limit = ViewSize;

#else /* Windows */

#ifndef MAP_TRYFIXED
#define MAP_TRYFIXED 0
#endif

#ifndef MAP_HASSEMAPHORE
#define MAP_HASSEMAPHORE 0
#endif

#ifndef MAP_CONCEAL
#define MAP_CONCEAL 0
#endif

#ifndef MAP_NOSYNC
#define MAP_NOSYNC 0
#endif

#ifndef MAP_FIXED_NOREPLACE
#define MAP_FIXED_NOREPLACE 0
#endif

#ifndef MAP_NORESERVE
#define MAP_NORESERVE 0
#endif

  map->base = mmap(nullptr, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ,
                   MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) |
                       ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC : MAP_CONCEAL),
                   map->fd, 0);

  if (unlikely(map->base == MAP_FAILED)) {
    map->limit = 0;
    map->current = 0;
    map->base = nullptr;
    ASSERT(errno != 0);
    return errno;
  }
  map->limit = limit;

#ifdef MADV_DONTFORK
  if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0))
    return errno;
#endif /* MADV_DONTFORK */
#ifdef MADV_NOHUGEPAGE
  (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE);
#endif /* MADV_NOHUGEPAGE */

#endif /* ! Windows */

  VALGRIND_MAKE_MEM_DEFINED(map->base, map->current);
  MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->current);
  return MDBX_SUCCESS;
}

void osal_munmap(osal_mmap_t *map) {
  VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
  /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
   * when this memory will re-used by malloc or another mmapping.
   * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */
  MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base,
                                   (map->filesize && map->filesize < map->limit) ? map->filesize : map->limit);
#if defined(_WIN32) || defined(_WIN64)
  if (map->section) {
    NtClose(map->section);
    map->section = 0;
  }
  NTSTATUS err = NtUnmapViewOfSection(GetCurrentProcess(), map->base);
  if (!NT_SUCCESS(err))
    ERROR("Unexpected NtUnmapViewOfSection(%p, %zi) error %d", map->base, map->limit, osal_ntstatus2errcode(err));
#else
  if (unlikely(munmap(map->base, map->limit)))
    ERROR("Unexpected munmap(%p, %zi) error %d", map->base, map->limit, errno);
#endif /* ! Windows */

  map->limit = 0;
  map->current = 0;
  map->base = nullptr;
}

int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit) {
  int rc = osal_filesize(map->fd, &map->filesize);
  VERBOSE("flags 0x%x, size %zu, limit %zu, filesize %" PRIu64, flags, size, limit, map->filesize);
  ASSERT(size <= limit);
  if (rc != MDBX_SUCCESS) {
    map->filesize = 0;
    return rc;
  }

#if defined(_WIN32) || defined(_WIN64)
  ASSERT(size != map->current || limit != map->limit || size < map->filesize);

  NTSTATUS status;
  LARGE_INTEGER SectionSize;
  int err;

  if (limit == map->limit && size > map->current) {
    if ((flags & MDBX_RDONLY) && map->filesize >= size) {
      map->current = size;
      return MDBX_SUCCESS;
    } else if (!(flags & MDBX_RDONLY) &&
               /* workaround for Wine */ imports.NtExtendSection) {
      /* growth rw-section */
      SectionSize.QuadPart = size;
      status = imports.NtExtendSection(map->section, &SectionSize);
      if (!NT_SUCCESS(status))
        return osal_ntstatus2errcode(status);
      map->current = size;
      if (map->filesize < size)
        map->filesize = size;
      return MDBX_SUCCESS;
    }
  }

  if (limit > map->limit) {
    err = check_mmap_limit(limit);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    /* check ability of address space for growth before unmap */
    PVOID BaseAddress = (PBYTE)map->base + map->limit;
    SIZE_T RegionSize = limit - map->limit;
    status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, &RegionSize, MEM_RESERVE, PAGE_NOACCESS);
    if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
      return MDBX_UNABLE_EXTEND_MAPSIZE;
    if (!NT_SUCCESS(status))
      return osal_ntstatus2errcode(status);

    status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, MEM_RELEASE);
    if (!NT_SUCCESS(status))
      return osal_ntstatus2errcode(status);
  }

  /* Windows unable:
   *  - shrink a mapped file;
   *  - change size of mapped view;
   *  - extend read-only mapping;
   * Therefore we should unmap/map entire section. */
  if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
    if (size <= map->current && limit == map->limit)
      return MDBX_SUCCESS;
    return MDBX_EPERM;
  }

  /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
   * when this memory will re-used by malloc or another mmapping.
   * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */
  MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->limit);
  status = NtUnmapViewOfSection(GetCurrentProcess(), map->base);
  if (!NT_SUCCESS(status))
    return osal_ntstatus2errcode(status);
  status = NtClose(map->section);
  map->section = nullptr;
  PVOID ReservedAddress = nullptr;
  SIZE_T ReservedSize = limit;

  if (!NT_SUCCESS(status)) {
  bailout_ntstatus:
    err = osal_ntstatus2errcode(status);
    map->base = nullptr;
    map->current = map->limit = 0;
    if (ReservedAddress) {
      ReservedSize = 0;
      status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE);
      ASSERT(NT_SUCCESS(status));
      (void)status;
    }
    return err;
  }

retry_file_and_section:
  /* resizing of the file may take a while,
   * therefore we reserve address space to avoid occupy it by other threads */
  ReservedAddress = map->base;
  status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, &ReservedSize, MEM_RESERVE, PAGE_NOACCESS);
  if (!NT_SUCCESS(status)) {
    ReservedAddress = nullptr;
    if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
      goto bailout_ntstatus /* no way to recovery */;

    if (flags & MDBX_MRESIZE_MAY_MOVE)
      /* the base address could be changed */
      map->base = nullptr;
  }

  if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) {
    err = osal_fsetsize(map->fd, size);
    if (err == MDBX_SUCCESS)
      map->filesize = size;
    /* ignore error, because Windows unable shrink file
     * that already mapped (by another process) */
  }

  SectionSize.QuadPart = size;
  status = NtCreateSection(&map->section,
                           /* DesiredAccess */
                           (flags & MDBX_WRITEMAP)
                               ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE
                               : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
                           /* ObjectAttributes */ nullptr,
                           /* MaximumSize (InitialSize) */ &SectionSize,
                           /* SectionPageProtection */
                           (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
                           /* AllocationAttributes */ SEC_RESERVE, map->fd);

  if (!NT_SUCCESS(status))
    goto bailout_ntstatus;

  if (ReservedAddress) {
    /* release reserved address space */
    ReservedSize = 0;
    status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE);
    ReservedAddress = nullptr;
    if (!NT_SUCCESS(status))
      goto bailout_ntstatus;
  }

retry_mapview:;
  SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit;
  status = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base,
                              /* ZeroBits */ 0,
                              /* CommitSize */ 0,
                              /* SectionOffset */ nullptr, &ViewSize,
                              /* InheritDisposition */ ViewUnmap,
                              /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
                              /* Win32Protect */
                              (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);

  if (!NT_SUCCESS(status)) {
    if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && map->base &&
        (flags & MDBX_MRESIZE_MAY_MOVE) != 0) {
      /* try remap at another base address */
      map->base = nullptr;
      goto retry_mapview;
    }
    NtClose(map->section);
    map->section = nullptr;

    if (map->base && (size != map->current || limit != map->limit)) {
      /* try remap with previously size and limit,
       * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */
      rc = (limit > map->limit) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM;
      size = map->current;
      ReservedSize = limit = map->limit;
      goto retry_file_and_section;
    }

    /* no way to recovery */
    goto bailout_ntstatus;
  }
  ASSERT(map->base != MAP_FAILED);

  map->current = (size_t)SectionSize.QuadPart;
  map->limit = ViewSize;

#else /* Windows */

  if (flags & MDBX_RDONLY) {
    if (size > map->filesize)
      rc = MDBX_UNABLE_EXTEND_MAPSIZE;
    else if (size < map->filesize && map->filesize > limit)
      rc = MDBX_EPERM;
    map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
  } else {
    if (map->filesize != size) {
      if (size > map->filesize) {
        rc = osal_fsetsize(map->fd, size);
        VERBOSE("osal_fsetsize-%s %zu, err %d", "extend", size, rc);
      } else if (flags & txn_shrink_allowed) {
        rc = osal_fsetsize(map->fd, size);
        VERBOSE("osal_fsetsize-%s %zu, err %d", "shrink", size, rc);
      }
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      map->filesize = size;
    }

    if (map->current > size) {
      /* Clearing asan's bitmask for the region which released in shrinking,
       * since:
       *  - after the shrinking we will get an exception when accessing
       *    this region and (therefore) do not need the help of ASAN.
       *  - this allows us to clear the mask only within the file size
       *    when closing the mapping. */
      MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr_disp(map->base, size),
                                       ((map->current < map->limit) ? map->current : map->limit) - size);
    }
    map->current = (size < map->limit) ? size : map->limit;
  }

  if (limit == map->limit)
    return rc;

  if (limit < map->limit) {
    /* unmap an excess at end of mapping. */
    // coverity[offset_free : FALSE]
    if (unlikely(munmap(ptr_disp(map->base, limit), map->limit - limit))) {
      ASSERT(errno != 0);
      return errno;
    }
    map->limit = limit;
    return rc;
  }

  int err = check_mmap_limit(limit);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  ASSERT(limit > map->limit);
  void *ptr = MAP_FAILED;

#if (defined(__linux__) || defined(__gnu_linux__)) && defined(_GNU_SOURCE)
  ptr = mremap(map->base, map->limit, limit,
#if defined(MREMAP_MAYMOVE)
               (flags & MDBX_MRESIZE_MAY_MOVE) ? MREMAP_MAYMOVE :
#endif /* MREMAP_MAYMOVE */
                                               0);
  if (ptr == MAP_FAILED) {
    err = errno;
    ASSERT(err != 0);
    switch (err) {
    default:
      return err;
    case 0 /* paranoia */:
    case EAGAIN:
    case ENOMEM:
      return MDBX_UNABLE_EXTEND_MAPSIZE;
    case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */:
      break;
    }
  }
#endif /* Linux & _GNU_SOURCE */

  const unsigned mmap_flags =
      MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0);
  const unsigned mmap_prot = (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ;

  if (ptr == MAP_FAILED) {
    /* Try to mmap additional space beyond the end of mapping. */
    ptr = mmap(ptr_disp(map->base, map->limit), limit - map->limit, mmap_prot, mmap_flags | MAP_FIXED_NOREPLACE,
               map->fd, map->limit);
    if (ptr == ptr_disp(map->base, map->limit))
      /* успешно прилепили отображение в конец */
      ptr = map->base;
    else if (ptr != MAP_FAILED) {
      /* the desired address is busy, unmap unsuitable one */
      if (unlikely(munmap(ptr, limit - map->limit))) {
        ASSERT(errno != 0);
        return errno;
      }
      ptr = MAP_FAILED;
    } else {
      err = errno;
      ASSERT(err != 0);
      switch (err) {
      default:
        return err;
      case 0 /* paranoia */:
      case EAGAIN:
      case ENOMEM:
        return MDBX_UNABLE_EXTEND_MAPSIZE;
      case EEXIST: /* address busy */
      case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */
        break;
      }
    }
  }

  if (ptr == MAP_FAILED) {
    /* unmap and map again whole region */
    if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
      /* TODO: Perhaps here it is worth to implement suspend/resume threads
       * and perform unmap/map as like for Windows. */
      return MDBX_UNABLE_EXTEND_MAPSIZE;
    }

    if (unlikely(munmap(map->base, map->limit))) {
      ASSERT(errno != 0);
      return errno;
    }

    // coverity[pass_freed_arg : FALSE]
    ptr = mmap(map->base, limit, mmap_prot,
               (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags
                                               : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED),
               map->fd, 0);
    if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) &&
        !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
      // coverity[pass_freed_arg : FALSE]
      ptr = mmap(map->base, limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0);

    if (unlikely(ptr == MAP_FAILED)) {
      /* try to restore prev mapping */
      // coverity[pass_freed_arg : FALSE]
      ptr = mmap(map->base, map->limit, mmap_prot,
                 (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags
                                                 : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED),
                 map->fd, 0);
      if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) &&
          !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
        // coverity[pass_freed_arg : FALSE]
        ptr = mmap(map->base, map->limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0);
      if (unlikely(ptr == MAP_FAILED)) {
        VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
        /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
         * when this memory will re-used by malloc or another mmapping.
         * See
         * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203
         */
        MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit);
        map->limit = 0;
        map->current = 0;
        map->base = nullptr;
        ASSERT(errno != 0);
        return errno;
      }
      rc = MDBX_UNABLE_EXTEND_MAPSIZE;
      limit = map->limit;
    }
  }

  ASSERT(ptr && ptr != MAP_FAILED);
  if (map->base != ptr) {
    VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
    /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
     * when this memory will re-used by malloc or another mmapping.
     * See
     * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203
     */
    MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit);

    VALGRIND_MAKE_MEM_DEFINED(ptr, map->current);
    MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current);
    map->base = ptr;
  }
  map->limit = limit;
  map->current = size;

#ifdef MADV_DONTFORK
  if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0)) {
    ASSERT(errno != 0);
    return errno;
  }
#endif /* MADV_DONTFORK */
#ifdef MADV_NOHUGEPAGE
  (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE);
#endif /* MADV_NOHUGEPAGE */

#endif /* POSIX / Windows */

  /* Zap: Redundant code */
  MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6287);
  ASSERT(rc != MDBX_SUCCESS || (map->base != nullptr && map->base != MAP_FAILED && map->current == size &&
                                map->limit == limit && map->filesize >= size));
  return rc;
}

/*----------------------------------------------------------------------------*/

__cold void osal_jitter(bool tiny) {
  for (;;) {
#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__)
    unsigned salt = 5296013u * (unsigned)__rdtsc();
    salt ^= salt >> 11;
    salt *= 25810541u;
#elif (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT
    static ULONG state;
    const unsigned salt = (unsigned)RtlRandomEx(&state);
#else
    const unsigned salt = rand();
#endif

    const int coin = salt % (tiny ? 29u : 43u);
    if (coin < 43 / 3)
      break;
#if defined(_WIN32) || defined(_WIN64)
    if (coin < 43 * 2 / 3)
      osal_yield();
    else {
      static HANDLE timer;
      if (!timer)
        timer = CreateWaitableTimer(NULL, TRUE, NULL);

      LARGE_INTEGER ft;
      ft.QuadPart = coin * (int64_t)-10; // Convert to 100 nanosecond interval,
                                         // negative value indicates relative time.
      SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0);
      WaitForSingleObject(timer, INFINITE);
      // CloseHandle(timer);
      break;
    }
#else
    osal_yield();
    if (coin > 43 * 2 / 3)
      usleep(coin);
#endif
  }
}

/*----------------------------------------------------------------------------*/

#if defined(_WIN32) || defined(_WIN64)
static LARGE_INTEGER performance_frequency;
#elif defined(__APPLE__) || defined(__MACH__)
#include <mach/mach_time.h>
static uint64_t ratio_16dot16_to_monotine;
#elif defined(__linux__) || defined(__gnu_linux__)
static clockid_t posix_clockid;
__cold static clockid_t choice_monoclock(void) {
  struct timespec probe;
#if defined(CLOCK_BOOTTIME)
  if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0)
    return CLOCK_BOOTTIME;
#elif defined(CLOCK_MONOTONIC_RAW)
  if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0)
    return CLOCK_MONOTONIC_RAW;
#elif defined(CLOCK_MONOTONIC_COARSE)
  if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0)
    return CLOCK_MONOTONIC_COARSE;
#endif
  return CLOCK_MONOTONIC;
}
#elif defined(CLOCK_MONOTONIC)
#define posix_clockid CLOCK_MONOTONIC
#else
#define posix_clockid CLOCK_REALTIME
#endif

uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16) {
#if defined(_WIN32) || defined(_WIN64)
  const uint64_t ratio = performance_frequency.QuadPart;
#elif defined(__APPLE__) || defined(__MACH__)
  const uint64_t ratio = ratio_16dot16_to_monotine;
#else
  const uint64_t ratio = UINT64_C(1000000000);
#endif
  const uint64_t ret = (ratio * seconds_16dot16 + 32768) >> 16;
  return likely(ret || seconds_16dot16 == 0) ? ret : /* fix underflow */ 1;
}

static uint64_t monotime_limit;
uint32_t osal_monotime_to_16dot16(uint64_t monotime) {
  if (unlikely(monotime > monotime_limit))
    return UINT32_MAX;

  const uint32_t ret =
#if defined(_WIN32) || defined(_WIN64)
      (uint32_t)((monotime << 16) / performance_frequency.QuadPart);
#elif defined(__APPLE__) || defined(__MACH__)
      (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine);
#else
      (uint32_t)((monotime << 7) / 1953125);
#endif
  return ret;
}

uint64_t osal_monotime(void) {
#if defined(_WIN32) || defined(_WIN64)
  LARGE_INTEGER counter;
  if (QueryPerformanceCounter(&counter))
    return counter.QuadPart;
#elif defined(__APPLE__) || defined(__MACH__)
  return mach_absolute_time();
#else
  struct timespec ts;
  if (likely(clock_gettime(posix_clockid, &ts) == 0))
    return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
#endif
  return 0;
}

uint64_t osal_cputime(size_t *optional_page_faults) {
#if defined(_WIN32) || defined(_WIN64)
  if (optional_page_faults) {
    PROCESS_MEMORY_COUNTERS pmc;
    *optional_page_faults = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) ? pmc.PageFaultCount : 0;
  }
  FILETIME unused, usermode;
  if (GetThreadTimes(GetCurrentThread(),
                     /* CreationTime */ &unused,
                     /* ExitTime */ &unused,
                     /* KernelTime */ &unused,
                     /* UserTime */ &usermode)) {
    /* one second = 10_000_000 * 100ns = 78125 * (1 << 7) * 100ns;
     * result = (h * f / 10_000_000) << 32) + l * f / 10_000_000 =
     *        = ((h * f) >> 7) / 78125) << 32) + ((l * f) >> 7) / 78125;
     * 1) {h, l} *= f;
     * 2) {h, l} >>= 7;
     * 3) result = ((h / 78125) << 32) + l / 78125; */
    uint64_t l = usermode.dwLowDateTime * performance_frequency.QuadPart;
    uint64_t h = usermode.dwHighDateTime * performance_frequency.QuadPart;
    l = h << (64 - 7) | l >> 7;
    h = h >> 7;
    return ((h / 78125) << 32) + l / 78125;
  }
#elif defined(RUSAGE_THREAD) || defined(RUSAGE_LWP)
#ifndef RUSAGE_THREAD
#define RUSAGE_THREAD RUSAGE_LWP /* Solaris */
#endif
  struct rusage usage;
  if (getrusage(RUSAGE_THREAD, &usage) == 0) {
    if (optional_page_faults)
      *optional_page_faults = usage.ru_majflt;
    return usage.ru_utime.tv_sec * UINT64_C(1000000000) + usage.ru_utime.tv_usec * 1000u;
  }
  if (optional_page_faults)
    *optional_page_faults = 0;
#elif defined(CLOCK_THREAD_CPUTIME_ID)
  if (optional_page_faults)
    *optional_page_faults = 0;
  struct timespec ts;
  if (likely(clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0))
    return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
#else
  /* FIXME */
  if (optional_page_faults)
    *optional_page_faults = 0;
#endif
  return 0;
}

/*----------------------------------------------------------------------------*/

static void bootid_shake(bin128_t *p) {
  /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */
  const uint32_t e = p->a - (p->b << 23 | p->b >> 9);
  p->a = p->b ^ (p->c << 16 | p->c >> 16);
  p->b = p->c + (p->d << 11 | p->d >> 21);
  p->c = p->d + e;
  p->d = e + p->a;
}

__cold static void bootid_collect(bin128_t *p, const void *s, size_t n) {
  p->y += UINT64_C(64526882297375213);
  bootid_shake(p);
  for (size_t i = 0; i < n; ++i) {
    bootid_shake(p);
    p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i];
    bootid_shake(p);
    p->y += 14621231;
  }
  bootid_shake(p);

  /* minor non-linear tomfoolery */
  const unsigned z = p->x % 61 + 1;
  p->y = p->y << z | p->y >> (64 - z);
  bootid_shake(p);
  bootid_shake(p);
  const unsigned q = p->x % 59 + 1;
  p->y = p->y << q | p->y >> (64 - q);
  bootid_shake(p);
  bootid_shake(p);
  bootid_shake(p);
}

static size_t hamming_weight(size_t v) {
  const size_t m1 = (size_t)UINT64_C(0x5555555555555555);
  const size_t m2 = (size_t)UINT64_C(0x3333333333333333);
  const size_t m4 = (size_t)UINT64_C(0x0f0f0f0f0f0f0f0f);
  const size_t h01 = (size_t)UINT64_C(0x0101010101010101);
  v -= (v >> 1) & m1;
  v = (v & m2) + ((v >> 2) & m2);
  v = (v + (v >> 4)) & m4;
  return (v * h01) >> (sizeof(v) * 8 - 8);
}

static inline size_t hw64(uint64_t v) {
  size_t r = hamming_weight((size_t)v);
  if (sizeof(v) > sizeof(r))
    r += hamming_weight((size_t)(v >> sizeof(r) * 4 >> sizeof(r) * 4));
  return r;
}

static bool check_uuid(bin128_t uuid) {
  size_t hw = hw64(uuid.x) + hw64(uuid.y) + hw64(uuid.x ^ uuid.y);
  return (hw >> 6) == 1;
}

#if defined(_WIN32) || defined(_WIN64)

__cold static uint64_t windows_systemtime_ms() {
  FILETIME ft;
  GetSystemTimeAsFileTime(&ft);
  return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul;
}

__cold static uint64_t windows_bootime(void) {
  unsigned confirmed = 0;
  uint64_t boottime = 0;
  uint64_t up0 = imports.GetTickCount64();
  uint64_t st0 = windows_systemtime_ms();
  for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) {
    YieldProcessor();
    const uint64_t up1 = imports.GetTickCount64();
    const uint64_t st1 = windows_systemtime_ms();
    if (st1 > fuse && st1 == st0 && up1 == up0) {
      uint64_t diff = st1 - up1;
      if (boottime == diff) {
        if (++confirmed > 4)
          return boottime;
      } else {
        confirmed = 0;
        boottime = diff;
      }
      fuse = st1;
      Sleep(1);
    }
    st0 = st1;
    up0 = up1;
  }
  return 0;
}

__cold static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue, PVOID pvData, LPDWORD pcbData) {
  LSTATUS rc;
  if (!imports.RegGetValueA) {
    /* an old Windows 2000/XP */
    HKEY hSubKey;
    rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey);
    if (rc == ERROR_SUCCESS) {
      rc = RegQueryValueExA(hSubKey, lpValue, nullptr, nullptr, pvData, pcbData);
      RegCloseKey(hSubKey);
    }
    return rc;
  }

  rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, nullptr, pvData, pcbData);
  if (rc != ERROR_FILE_NOT_FOUND)
    return rc;

  rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, nullptr,
                            pvData, pcbData);
  if (rc != ERROR_FILE_NOT_FOUND)
    return rc;
  return imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, nullptr,
                              pvData, pcbData);
}
#endif

MDBX_MAYBE_UNUSED __cold static bool bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) {
  if (n > 31) {
    unsigned bits = 0;
    for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ {
      uint8_t c = ((const uint8_t *)p)[i];
      if (c >= '0' && c <= '9')
        c -= '0';
      else if (c >= 'a' && c <= 'f')
        c -= 'a' - 10;
      else if (c >= 'A' && c <= 'F')
        c -= 'A' - 10;
      else
        continue;
      ASSERT(c <= 15);
      c ^= s->y >> 60;
      s->y = s->y << 4 | s->x >> 60;
      s->x = s->x << 4 | c;
      bits += 4;
    }
    if (bits > 42 * 3)
      /* UUID parsed successfully */
      return true;
  }

  if (n > 15) /* is enough handle it as a binary? */ {
    if (n == sizeof(bin128_t)) {
      bin128_t aligned;
      memcpy(&aligned, p, sizeof(bin128_t));
      s->x += aligned.x;
      s->y += aligned.y;
    } else
      bootid_collect(s, p, n);
    return check_uuid(*s);
  }

  if (n)
    bootid_collect(s, p, n);
  return false;
}

#if defined(__linux__) || defined(__gnu_linux__)

__cold static bool is_inside_lxc(void) {
  bool inside_lxc = false;
  FILE *mounted = setmntent("/proc/mounts", "r");
  if (mounted) {
    const struct mntent *ent;
    while (nullptr != (ent = getmntent(mounted))) {
      if (strcmp(ent->mnt_fsname, "lxcfs") == 0 && strncmp(ent->mnt_dir, "/proc/", 6) == 0) {
        inside_lxc = true;
        break;
      }
    }
    endmntent(mounted);
  }
  return inside_lxc;
}

__cold static bool proc_read_uuid(const char *path, bin128_t *target) {
  const int fd = open(path, O_RDONLY | O_NOFOLLOW);
  if (fd != -1) {
    struct statfs fs;
    char buf[42];
    const ssize_t len = (fstatfs(fd, &fs) == 0 &&
                         (fs.f_type == /* procfs */ 0x9FA0 || (fs.f_type == /* tmpfs */ 0x1021994 && is_inside_lxc())))
                            ? read(fd, buf, sizeof(buf))
                            : -1;
    const int err = close(fd);
    ASSERT(err == 0);
    (void)err;
    if (len > 0)
      return bootid_parse_uuid(target, buf, len);
  }
  return false;
}
#endif /* Linux */

__cold static bin128_t osal_bootid(void) {
  bin128_t uuid = {{0, 0}};
  bool got_machineid = false, got_boottime = false, got_bootseq = false;

#if defined(__linux__) || defined(__gnu_linux__)
  if (proc_read_uuid("/proc/sys/kernel/random/boot_id", &uuid))
    return uuid;
#endif /* Linux */

#if defined(__APPLE__) || defined(__MACH__)
  {
    char buf[42];
    size_t len = sizeof(buf);
    if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && bootid_parse_uuid(&uuid, buf, len))
      return uuid;

#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED > 1050
    uuid_t hostuuid;
    struct timespec wait = {0, 1000000000u / 42};
    if (!gethostuuid(hostuuid, &wait))
      got_machineid = bootid_parse_uuid(&uuid, hostuuid, sizeof(hostuuid));
#endif /* > 10.5 */

    struct timeval boottime;
    len = sizeof(boottime);
    if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && len == sizeof(boottime) && boottime.tv_sec)
      got_boottime = true;
  }
#endif /* Apple/Darwin */

#if defined(_WIN32) || defined(_WIN64)
  {
    union buf {
      DWORD BootId;
      DWORD BaseTime;
      SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo;
      struct {
        LARGE_INTEGER BootTime;
        LARGE_INTEGER CurrentTime;
        LARGE_INTEGER TimeZoneBias;
        ULONG TimeZoneId;
        ULONG Reserved;
        ULONGLONG BootTimeBias;
        ULONGLONG SleepTimeBias;
      } SysTimeOfDayInfoHacked;
      wchar_t MachineGuid[42];
      char DigitalProductId[248];
    } buf;

    static const char HKLM_MicrosoftCryptography[] = "SOFTWARE\\Microsoft\\Cryptography";
    DWORD len = sizeof(buf);
    /* Windows is madness and must die */
    if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, "MachineGuid", &buf.MachineGuid, &len) ==
            ERROR_SUCCESS &&
        len < sizeof(buf))
      got_machineid = bootid_parse_uuid(&uuid, &buf.MachineGuid, len);

    if (!got_machineid) {
      /* again, Windows is madness */
      static const char HKLM_WindowsNT[] = "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion";
      static const char HKLM_WindowsNT_DPK[] = "SOFTWARE\\Microsoft\\Windows "
                                               "NT\\CurrentVersion\\DefaultProductKey";
      static const char HKLM_WindowsNT_DPK2[] = "SOFTWARE\\Microsoft\\Windows "
                                                "NT\\CurrentVersion\\DefaultProductKey2";

      len = sizeof(buf);
      if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, "DigitalProductId", &buf.DigitalProductId, &len) ==
              ERROR_SUCCESS &&
          len > 42 && len < sizeof(buf)) {
        bootid_collect(&uuid, &buf.DigitalProductId, len);
        got_machineid = true;
      }
      len = sizeof(buf);
      if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, "DigitalProductId", &buf.DigitalProductId, &len) ==
              ERROR_SUCCESS &&
          len > 42 && len < sizeof(buf)) {
        bootid_collect(&uuid, &buf.DigitalProductId, len);
        got_machineid = true;
      }
      len = sizeof(buf);
      if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, "DigitalProductId", &buf.DigitalProductId, &len) ==
              ERROR_SUCCESS &&
          len > 42 && len < sizeof(buf)) {
        bootid_collect(&uuid, &buf.DigitalProductId, len);
        got_machineid = true;
      }
    }

    static const char HKLM_PrefetcherParams[] = "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory "
                                                "Management\\PrefetchParameters";
    len = sizeof(buf);
    if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", &buf.BootId, &len) == ERROR_SUCCESS &&
        len > 1 && len < sizeof(buf)) {
      bootid_collect(&uuid, &buf.BootId, len);
      got_bootseq = true;
    }

    len = sizeof(buf);
    if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", &buf.BaseTime, &len) == ERROR_SUCCESS &&
        len >= sizeof(buf.BaseTime) && buf.BaseTime) {
      bootid_collect(&uuid, &buf.BaseTime, len);
      got_boottime = true;
    }

    /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */
    NTSTATUS status = NtQuerySystemInformation(0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo,
                                               sizeof(buf.SysTimeOfDayInfo), &len);
    if (NT_SUCCESS(status) &&
        len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) +
                   sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) &&
        buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) {
      const uint64_t UnbiasedBootTime =
          buf.SysTimeOfDayInfoHacked.BootTime.QuadPart - buf.SysTimeOfDayInfoHacked.BootTimeBias;
      if (UnbiasedBootTime) {
        bootid_collect(&uuid, &UnbiasedBootTime, sizeof(UnbiasedBootTime));
        got_boottime = true;
      }
    }

    if (!got_boottime) {
      uint64_t boottime = windows_bootime();
      if (boottime) {
        bootid_collect(&uuid, &boottime, sizeof(boottime));
        got_boottime = true;
      }
    }
  }
#endif /* Windows */

#if defined(CTL_HW) && defined(HW_UUID)
  if (!got_machineid) {
    static const int mib[] = {CTL_HW, HW_UUID};
    char buf[42];
    size_t len = sizeof(buf);
    if (sysctl(
#ifdef SYSCTL_LEGACY_NONCONST_MIB
            (int *)
#endif
                mib,
            ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0)
      got_machineid = bootid_parse_uuid(&uuid, buf, len);
  }
#endif /* CTL_HW && HW_UUID */

#if defined(CTL_KERN) && defined(KERN_HOSTUUID)
  if (!got_machineid) {
    static const int mib[] = {CTL_KERN, KERN_HOSTUUID};
    char buf[42];
    size_t len = sizeof(buf);
    if (sysctl(
#ifdef SYSCTL_LEGACY_NONCONST_MIB
            (int *)
#endif
                mib,
            ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0)
      got_machineid = bootid_parse_uuid(&uuid, buf, len);
  }
#endif /* CTL_KERN && KERN_HOSTUUID */

#if defined(__NetBSD__)
  if (!got_machineid) {
    char buf[42];
    size_t len = sizeof(buf);
    if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, nullptr, 0) == 0)
      got_machineid = bootid_parse_uuid(&uuid, buf, len);
  }
#endif /* __NetBSD__ */

#if !(defined(_WIN32) || defined(_WIN64))
  if (!got_machineid) {
    int fd = open("/etc/machine-id", O_RDONLY);
    if (fd == -1)
      fd = open("/var/lib/dbus/machine-id", O_RDONLY);
    if (fd != -1) {
      char buf[42];
      const ssize_t len = read(fd, buf, sizeof(buf));
      const int err = close(fd);
      ASSERT(err == 0);
      (void)err;
      if (len > 0)
        got_machineid = bootid_parse_uuid(&uuid, buf, len);
    }
  }
#endif /* !Windows */

#if _XOPEN_SOURCE_EXTENDED
  if (!got_machineid) {
    const long hostid = gethostid();
    if (hostid != 0 && hostid != -1) {
      bootid_collect(&uuid, &hostid, sizeof(hostid));
      got_machineid = true;
    }
  }
#endif /* _XOPEN_SOURCE_EXTENDED */

  if (!got_machineid) {
  lack:
    uuid.x = uuid.y = 0;
    return uuid;
  }

  /*--------------------------------------------------------------------------*/

#if defined(CTL_KERN) && defined(KERN_BOOTTIME)
  if (!got_boottime) {
    static const int mib[] = {CTL_KERN, KERN_BOOTTIME};
    struct timeval boottime;
    size_t len = sizeof(boottime);
    if (sysctl(
#ifdef SYSCTL_LEGACY_NONCONST_MIB
            (int *)
#endif
                mib,
            ARRAY_LENGTH(mib), &boottime, &len, nullptr, 0) == 0 &&
        len == sizeof(boottime) && boottime.tv_sec) {
      bootid_collect(&uuid, &boottime, len);
      got_boottime = true;
    }
  }
#endif /* CTL_KERN && KERN_BOOTTIME */

#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
  if (!got_boottime) {
    kstat_ctl_t *kc = kstat_open();
    if (kc) {
      kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc");
      if (kp && kstat_read(kc, kp, 0) != -1) {
        kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time");
        if (kn) {
          switch (kn->data_type) {
          case KSTAT_DATA_INT32:
          case KSTAT_DATA_UINT32:
            bootid_collect(&uuid, &kn->value, sizeof(int32_t));
            got_boottime = true;
            break;
          case KSTAT_DATA_INT64:
          case KSTAT_DATA_UINT64:
            bootid_collect(&uuid, &kn->value, sizeof(int64_t));
            got_boottime = true;
            break;
          }
        }
      }
      kstat_close(kc);
    }
  }
#endif /* SunOS / Solaris */

#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME)
  if (!got_boottime) {
    setutxent();
    const struct utmpx id = {.ut_type = BOOT_TIME};
    const struct utmpx *entry = getutxid(&id);
    if (entry) {
      bootid_collect(&uuid, entry, sizeof(*entry));
      got_boottime = true;
      while (unlikely((entry = getutxid(&id)) != nullptr)) {
        /* have multiple reboot records, assuming we can distinguish next
         * bootsession even if RTC is wrong or absent */
        bootid_collect(&uuid, entry, sizeof(*entry));
        got_bootseq = true;
      }
    }
    endutxent();
  }
#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */

  if (!got_bootseq) {
    if (!got_boottime || !MDBX_TRUST_RTC)
      goto lack;

#if defined(_WIN32) || defined(_WIN64)
    FILETIME now;
    GetSystemTimeAsFileTime(&now);
    if (0x1CCCCCC > now.dwHighDateTime)
#else
    struct timespec mono, real;
    if (clock_gettime(CLOCK_MONOTONIC, &mono) || clock_gettime(CLOCK_REALTIME, &real) ||
        /* wrong time, RTC is mad or absent */
        1555555555l > real.tv_sec ||
        /* seems no adjustment by RTC/NTP, i.e. a fake time */
        real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || (real.tv_sec - mono.tv_sec) % 900u == 0)
#endif
      goto lack;
  }

  return uuid;
}

__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, intptr_t *avail_pages) {
  if (!page_size && !total_pages && !avail_pages)
    return LOG_IFERR(MDBX_EINVAL);
  if (total_pages)
    *total_pages = -1;
  if (avail_pages)
    *avail_pages = -1;

  const intptr_t pagesize = globals.sys_pagesize;
  if (page_size)
    *page_size = pagesize;
  if (unlikely(pagesize < MDBX_MIN_PAGESIZE || !is_powerof2(pagesize)))
    return LOG_IFERR(MDBX_INCOMPATIBLE);

  const int log2page = log2n_powerof2(pagesize);
  ASSERT(pagesize == (INT64_C(1) << log2page));
  (void)log2page;

#if defined(_WIN32) || defined(_WIN64)
  MEMORYSTATUSEX info;
  memset(&info, 0, sizeof(info));
  info.dwLength = sizeof(info);
  if (!GlobalMemoryStatusEx(&info))
    return LOG_IFERR((int)GetLastError());
#endif

  if (total_pages) {
#if defined(_WIN32) || defined(_WIN64)
    const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page);
#elif defined(_SC_PHYS_PAGES)
    const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES);
    if (total_ram_pages == -1)
      return LOG_IFERR(errno);
#elif defined(_SC_AIX_REALMEM)
    const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM);
    if (total_ram_Kb == -1)
      return LOG_IFERR(errno);
    const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page;
#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || defined(HW_PHYSMEM)
    size_t ram, len = sizeof(ram);
    static const int mib[] = {CTL_HW,
#if defined(HW_USERMEM)
                              HW_USERMEM
#elif defined(HW_PHYSMEM64)
                              HW_PHYSMEM64
#elif defined(HW_MEMSIZE)
                              HW_MEMSIZE
#else
                              HW_PHYSMEM
#endif
    };
    if (sysctl(
#ifdef SYSCTL_LEGACY_NONCONST_MIB
            (int *)
#endif
                mib,
            ARRAY_LENGTH(mib), &ram, &len, nullptr, 0) != 0)
      return LOG_IFERR(errno);
    if (len != sizeof(ram))
      return LOG_IFERR(MDBX_ENOSYS);
    const intptr_t total_ram_pages = (intptr_t)(ram >> log2page);
#else
#error "FIXME: Get User-accessible or physical RAM"
#endif
    *total_pages = total_ram_pages;
    if (total_ram_pages < 1)
      return LOG_IFERR(MDBX_ENOSYS);
  }

  if (avail_pages) {
#if defined(_WIN32) || defined(_WIN64)
    const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page);
#elif defined(_SC_AVPHYS_PAGES)
    const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES);
    if (avail_ram_pages == -1)
      return LOG_IFERR(errno);
#elif defined(__MACH__)
    mach_msg_type_number_t count = HOST_VM_INFO_COUNT;
    vm_statistics_data_t vmstat;
    mach_port_t mport = mach_host_self();
    kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t)&vmstat, &count);
    mach_port_deallocate(mach_task_self(), mport);
    if (unlikely(kerr != KERN_SUCCESS))
      return LOG_IFERR(MDBX_ENOSYS);
    const intptr_t avail_ram_pages = vmstat.free_count;
#elif defined(VM_TOTAL) || defined(VM_METER)
    struct vmtotal info;
    size_t len = sizeof(info);
    static const int mib[] = {CTL_VM,
#if defined(VM_TOTAL)
                              VM_TOTAL
#elif defined(VM_METER)
                              VM_METER
#endif
    };
    if (sysctl(
#ifdef SYSCTL_LEGACY_NONCONST_MIB
            (int *)
#endif
                mib,
            ARRAY_LENGTH(mib), &info, &len, nullptr, 0) != 0)
      return LOG_IFERR(errno);
    if (len != sizeof(info))
      return LOG_IFERR(MDBX_ENOSYS);
    const intptr_t avail_ram_pages = info.t_free;
#else
#error "FIXME: Get Available RAM"
#endif
    *avail_pages = avail_ram_pages;
    if (avail_ram_pages < 1)
      return LOG_IFERR(MDBX_ENOSYS);
  }

  return MDBX_SUCCESS;
}

/*----------------------------------------------------------------------------*/

#ifdef __FreeBSD__
#include <sys/uuid.h>
#endif /* FreeBSD */

#ifdef __IPHONE_OS_VERSION_MIN_REQUIRED
#include <CommonCrypto/CommonRandom.h>
#elif __GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \
    defined(__DragonFly__) || defined(__APPLE__) || __has_include(<sys/random.h>)
#include <sys/random.h>
#endif /* sys/random.h */

#if defined(_WIN32) || defined(_WIN64)
#include <wincrypt.h>
#endif /* Windows */

bin128_t osal_guid(const MDBX_env *env) {
  struct {
    uint64_t begin, end, cputime;
    uintptr_t thread, pid;
    const void *x, *y;
    bin128_t (*z)(const MDBX_env *env);
  } salt;

  salt.begin = osal_monotime();
  bin128_t uuid = {{0, 0}};

#if defined(__linux__) || defined(__gnu_linux__)
  if (proc_read_uuid("/proc/sys/kernel/random/uuid", &uuid) && check_uuid(uuid))
    return uuid;
#endif /* Linux */

#ifdef __FreeBSD__
  STATIC_ASSERT(sizeof(uuid) == sizeof(struct uuid));
  if (uuidgen((struct uuid *)&uuid, 1) == 0 && check_uuid(uuid))
    return uuid;
#endif /* FreeBSD */

#if defined(_WIN32) || defined(_WIN64)
  if (imports.CoCreateGuid && imports.CoCreateGuid(&uuid) == 0 && check_uuid(uuid))
    return uuid;

  HCRYPTPROV hCryptProv = 0;
  if (CryptAcquireContextW(&hCryptProv, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) {
    const BOOL ok = CryptGenRandom(hCryptProv, sizeof(uuid), (unsigned char *)&uuid);
    CryptReleaseContext(hCryptProv, 0);
    if (ok && check_uuid(uuid))
      return uuid;
  }
#elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_8_0)
#if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0
  if (CCRandomGenerateBytes(&uuid, sizeof(uuid)) == kCCSuccess && check_uuid(uuid))
    return uuid;
#endif /* iOS >= 8.x */
#else
  const int fd = open("/dev/urandom", O_RDONLY);
  if (fd != -1) {
    const ssize_t len = read(fd, &uuid, sizeof(uuid));
    const int err = close(fd);
    ASSERT(err == 0);
    (void)err;
    if (len == sizeof(uuid) && check_uuid(uuid))
      return uuid;
  }
#if (__GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) ||  \
     defined(__DragonFly__)) &&                                                                                        \
    !defined(__APPLE__) && !defined(__ANDROID_API__)
  if (getrandom(&uuid, sizeof(uuid), 0) == sizeof(uuid) && check_uuid(uuid))
    return uuid;
#elif defined(__OpenBSD__) || (defined(__sun) && defined(__SVR4)) ||                                                   \
    (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
  if (getentropy(&uuid, sizeof(uuid)) == 0 && check_uuid(uuid))
    return uuid;
#endif /* getrandom() / getentropy() */
#endif /* !Windows */

  uuid = globals.bootid;
  bootid_collect(&uuid, env, sizeof(*env));
  salt.thread = osal_thread_self();
  salt.pid = osal_getpid();
  salt.x = &salt;
  salt.y = env;
  salt.z = &osal_guid;
  do {
    salt.cputime = osal_cputime(nullptr);
    salt.end = osal_monotime();
    bootid_collect(&uuid, &salt, sizeof(salt));
  } while (!check_uuid(uuid));
  return uuid;
}

const char *osal_getenv(const char *name, bool secure) {
  (void)secure;
#if defined(_WIN32) || defined(_WIN64)
  static char buf[42];
  SetLastError(ERROR_OUT_OF_PAPER);
  const size_t len = GetEnvironmentVariableA(name, buf, sizeof(buf));
  if (len >= sizeof(buf))
    /* no idea haw to handle */
    return nullptr;
  if (len != 0)
    return buf;
  switch (GetLastError()) {
  case ERROR_OUT_OF_PAPER:
    return "";
  default:
    /* no idea to do in case of other error */
  case ERROR_ENVVAR_NOT_FOUND:
    return nullptr;
  }
  return (GetLastError() == ERROR_ENVVAR_NOT_FOUND) ? nullptr : "";
#else
#if defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 17)
  if (secure)
    return secure_getenv(name);
#endif /* glibc >= 2.17 */
  return getenv(name);
#endif
}

bool osal_safe_peek_uint32(const void *ptr, int32_t *dest) {
  bool done = false;
  *dest = 0;
#if defined(_WIN32) || defined(_WIN64)
  __try {
    if (IsBadReadPtr(ptr, 4) == 0) {
      memcpy(dest, ptr, 4);
      done = true;
    }
  } __except (EXCEPTION_EXECUTE_HANDLER) {
    return false;
  }
#else
  static int nullfd = -1;
  if (nullfd < 0) {
    nullfd = open(dev_null, O_WRONLY
#ifdef O_CLOEXEC
                                | O_CLOEXEC
#endif /* O_CLOEXEC */
    );
    if (unlikely(nullfd < 0)) {
      ERROR("unable open(%s), err %d", dev_null, errno);
      return false;
    }
  }
  if (write(nullfd, ptr, 4) == 4) {
    memcpy(dest, ptr, 4);
    done = true;
  }
#endif
  return done;
}

/*--------------------------------------------------------------------------*/

void osal_ctor(void) {
#if MDBX_HAVE_PWRITEV && (defined(_SC_IOV_MAX) || defined(ENABLE_MEMCHECK))
#if defined(_SC_IOV_MAX)
  osal_iov_max = (_SC_IOV_MAX);
#else
  osal_iov_max = IOV_MAX;
#endif /* _SC_IOV_MAX */
  if (RUNNING_ON_VALGRIND && osal_iov_max > 64)
    /* чтобы не описывать все 1024 исключения в valgrind_suppress.txt */
    osal_iov_max = 64;
#endif /* MDBX_HAVE_PWRITEV */

#if defined(_WIN32) || defined(_WIN64)
  SYSTEM_INFO si;
  GetSystemInfo(&si);
  globals.sys_pagesize = si.dwPageSize;
  globals.sys_allocation_granularity =
      (si.dwAllocationGranularity > globals.sys_pagesize) ? si.dwAllocationGranularity : globals.sys_pagesize;
#else
  globals.sys_pagesize = sysconf(_SC_PAGE_SIZE);
  globals.sys_allocation_granularity = (MDBX_WORDBITS > 32) ? 65536 : 16384;
  globals.sys_allocation_granularity = (globals.sys_allocation_granularity >= globals.sys_pagesize * 2)
                                           ? globals.sys_allocation_granularity
                                           : globals.sys_pagesize * 4;
#ifdef AT_UCACHEBSIZE
  const size_t unified_cache_block_size = getauxval(AT_UCACHEBSIZE);
  globals.sys_unified_cache_block = globals.sys_pagesize;
  if (unified_cache_block_size > 0 && unified_cache_block_size < INT_MAX) {
    globals.sys_unified_cache_block = (unsigned)unified_cache_block_size;
    if (globals.sys_unified_cache_block > globals.sys_pagesize)
      globals.sys_allocation_granularity = globals.sys_unified_cache_block;
  }
#endif /* AT_UCACHEBSIZE */
#endif
  if (globals.sys_allocation_granularity > 4 * MEGABYTE && globals.sys_pagesize < MEGABYTE)
    globals.sys_allocation_granularity = 4 * MEGABYTE;
  ASSERT(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0);
  ASSERT(globals.sys_allocation_granularity >= globals.sys_pagesize &&
         globals.sys_allocation_granularity % globals.sys_pagesize == 0);
  globals.sys_pagesize_ln2 = log2n_powerof2(globals.sys_pagesize);

#if defined(__linux__) || defined(__gnu_linux__)
  posix_clockid = choice_monoclock();
#endif

#if defined(_WIN32) || defined(_WIN64)
  QueryPerformanceFrequency(&performance_frequency);
#elif defined(__APPLE__) || defined(__MACH__)
  mach_timebase_info_data_t ti;
  mach_timebase_info(&ti);
  ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer;
#endif
  monotime_limit = osal_16dot16_to_monotime(UINT32_MAX - 1);

  uint32_t proba = UINT32_MAX;
  while (true) {
    unsigned time_conversion_checkup = osal_monotime_to_16dot16(osal_16dot16_to_monotime(proba));
    unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba;
    unsigned one_less = (proba > 0) ? proba - 1 : proba;
    ENSURE(time_conversion_checkup >= one_less && time_conversion_checkup <= one_more);
    if (proba == 0)
      break;
    proba >>= 1;
  }

  globals.bootid = osal_bootid();
}

void osal_dtor(void) {}

__cold int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...) {
  if (LOG_ENABLED(MDBX_LOG_ERROR)) {
    static const page_t *prev;
    if (prev != mp) {
      char buf4unknown[16];
      prev = mp;
      debug_log(MDBX_LOG_ERROR, "badpage", 0, "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n",
                pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid);
    }

    va_list args;
    va_start(args, fmt);
    debug_log_va(MDBX_LOG_ERROR, "badpage", 0, fmt, args);
    va_end(args);
  }
  return MDBX_CORRUPTED;
}

__cold void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...) {
  if (LOG_ENABLED(MDBX_LOG_NOTICE)) {
    static const page_t *prev;
    if (prev != mp) {
      char buf4unknown[16];
      prev = mp;
      debug_log(MDBX_LOG_NOTICE, "poorpage", 0, "suboptimal %s-page #%u, mod-txnid %" PRIaTXN "\n",
                pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid);
    }

    va_list args;
    va_start(args, fmt);
    debug_log_va(MDBX_LOG_NOTICE, "poorpage", 0, fmt, args);
    va_end(args);
  }
}

MDBX_CONST_FUNCTION static clc_t value_clc(const MDBX_cursor *mc) {
  if (likely((mc->flags & z_inner) == 0))
    return mc->clc->v;
  else {
    clc_t stub = {.cmp = cmp_equal_or_wrong, .lmin = 0, .lmax = 0};
    return stub;
  }
}

__cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
  DKBUF;
  int rc = MDBX_SUCCESS;
  if (unlikely(mp->pgno < MIN_PAGENO || mp->pgno > MAX_PAGENO))
    rc = bad_page(mp, "invalid pgno (%u)\n", mp->pgno);

  MDBX_env *const env = mc->txn->env;
  const ptrdiff_t offset = ptr_dist(mp, env->dxb_mmap.base);
  unsigned flags_mask = P_ILL_BITS;
  unsigned flags_expected = 0;
  if (offset < 0 || offset > (ptrdiff_t)(pgno2bytes(env, mc->txn->geo.first_unallocated) -
                                         ((mp->flags & P_SUBP) ? PAGEHDRSZ + 1 : env->ps))) {
    /* should be dirty page without MDBX_WRITEMAP, or a subpage of. */
    flags_mask -= P_SUBP;
    if ((env->flags & MDBX_WRITEMAP) != 0 || (!is_shadowed(mc->txn, mp) && !(mp->flags & P_SUBP)))
      rc = bad_page(mp, "invalid page-address %p, offset %zi\n", __Wpedantic_format_voidptr(mp), offset);
  } else if (offset & (env->ps - 1))
    flags_expected = P_SUBP;

  if (unlikely((mp->flags & flags_mask) != flags_expected))
    rc = bad_page(mp, "unknown/extra page-flags (have 0x%x, expect 0x%x)\n", mp->flags & flags_mask, flags_expected);

  cASSERT0(mc, (mc->checking & z_dupfix) == 0 || (mc->flags & z_inner) != 0);
  const uint8_t type = page_type(mp);
  switch (type) {
  default:
    return bad_page(mp, "invalid type (%u)\n", type);
  case P_LARGE:
    if (unlikely(mc->flags & z_inner))
      rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "large", "nested dupsort tree", mc->tree->flags);
    const pgno_t npages = mp->pages;
    if (unlikely(npages < 1 || npages >= MAX_PAGENO / 2))
      rc = bad_page(mp, "invalid n-pages (%u) for large-page\n", npages);
    if (unlikely(mp->pgno + npages > mc->txn->geo.first_unallocated))
      rc = bad_page(mp, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", mp->pgno + npages,
                    mc->txn->geo.first_unallocated);
    return rc; //-------------------------- end of large/overflow page handling
  case P_LEAF | P_SUBP:
    if (unlikely(mc->tree->height != 1))
      rc =
          bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf-sub", "nested dupsort db", mc->tree->flags);
    /* fall through */
    __fallthrough;
  case P_LEAF:
    if (unlikely((mc->checking & z_dupfix) != 0))
      rc = bad_page(mp, "unexpected leaf-page for dupfix subtree (db-lags 0x%x)\n", mc->tree->flags);
    break;
  case P_LEAF | P_DUPFIX | P_SUBP:
    if (unlikely(mc->tree->height != 1))
      rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf2-sub", "nested dupsort db",
                    mc->tree->flags);
    /* fall through */
    __fallthrough;
  case P_LEAF | P_DUPFIX:
    if (unlikely((mc->checking & z_dupfix) == 0))
      rc = bad_page(mp, "unexpected leaf2-page for non-dupfix (sub)tree (db-flags 0x%x)\n", mc->tree->flags);
    break;
  case P_BRANCH:
    break;
  }

  if (unlikely(mp->upper < mp->lower || (mp->lower & 1) || PAGEHDRSZ + mp->upper > env->ps))
    rc = bad_page(mp, "invalid page lower(%u)/upper(%u) with limit %zu\n", mp->lower, mp->upper, page_space(env));

  const char *const end_of_page = ptr_disp(mp, env->ps);
  const size_t nkeys = page_numkeys(mp);
  STATIC_ASSERT(P_BRANCH == 1);
  if (unlikely(nkeys <= (uint8_t)(mp->flags & P_BRANCH))) {
    if ((!(mc->flags & z_inner) || mc->tree->items) &&
        (!(mc->checking & z_updating) || !(is_modifable(mc->txn, mp) || (mp->flags & P_SUBP))))
      rc = bad_page(mp, "%s-page nkeys (%zu) < %u\n", is_branch(mp) ? "branch" : "leaf", nkeys, 1 + is_branch(mp));
  }

  const size_t ksize_max = keysize_max(env->ps, 0);
  const size_t leaf2_ksize = mp->dupfix_ksize;
  if (is_dupfix_leaf(mp)) {
    if (unlikely((mc->flags & z_inner) == 0 || (mc->tree->flags & MDBX_DUPFIXED) == 0))
      rc = bad_page(mp, "unexpected leaf2-page (db-flags 0x%x)\n", mc->tree->flags);
    else if (unlikely(leaf2_ksize != mc->tree->dupfix_size))
      rc = bad_page(mp, "invalid leaf2_ksize %zu\n", leaf2_ksize);
    else if (unlikely(((leaf2_ksize & nkeys) ^ mp->upper) & 1))
      rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with leaf2-length %zu\n", mp->upper, nkeys, leaf2_ksize);
  } else {
    if (unlikely((mp->upper & 1) || PAGEHDRSZ + mp->upper + nkeys * sizeof(node_t) + nkeys - 1 > env->ps))
      rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with limit %zu\n", mp->upper, nkeys, page_space(env));
  }

  MDBX_val here, prev = {0, 0};
  clc_t v_clc = value_clc(mc);
  for (size_t i = 0; i < nkeys; ++i) {
    if (is_dupfix_leaf(mp)) {
      const char *const key = page_dupfix_ptr(mp, i, mc->tree->dupfix_size);
      if (unlikely(end_of_page < key + leaf2_ksize)) {
        rc = bad_page(mp, "leaf2-item beyond (%zu) page-end\n", key + leaf2_ksize - end_of_page);
        continue;
      }

      if (unlikely(leaf2_ksize != mc->clc->k.lmin)) {
        if (unlikely(leaf2_ksize < mc->clc->k.lmin || leaf2_ksize > mc->clc->k.lmax))
          rc = bad_page(mp, "leaf2-item size (%zu) <> min/max length (%zu/%zu)\n", leaf2_ksize, mc->clc->k.lmin,
                        mc->clc->k.lmax);
        else
          mc->clc->k.lmin = mc->clc->k.lmax = leaf2_ksize;
      }
      if ((mc->checking & z_ignord) == 0) {
        here.iov_base = (void *)key;
        here.iov_len = leaf2_ksize;
        if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0))
          rc = bad_page(mp, "leaf2-item #%zu wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here));
        prev = here;
      }
    } else {
      const node_t *const node = page_node(mp, i);
      const char *const node_end = ptr_disp(node, NODESIZE);
      if (unlikely(node_end > end_of_page)) {
        rc = bad_page(mp, "node[%zu] (%zu) beyond page-end\n", i, node_end - end_of_page);
        continue;
      }
      const size_t ksize = node_ks(node);
      if (unlikely(ksize > ksize_max))
        rc = bad_page(mp, "node[%zu] too long key (%zu)\n", i, ksize);
      const char *const key = node_key(node);
      if (unlikely(end_of_page < key + ksize)) {
        rc = bad_page(mp, "node[%zu] key (%zu) beyond page-end\n", i, key + ksize - end_of_page);
        continue;
      }
      if ((is_leaf(mp) || i > 0)) {
        if (unlikely(ksize < mc->clc->k.lmin || ksize > mc->clc->k.lmax))
          rc = bad_page(mp, "node[%zu] key size (%zu) <> min/max key-length (%zu/%zu)\n", i, ksize, mc->clc->k.lmin,
                        mc->clc->k.lmax);
        if ((mc->checking & z_ignord) == 0) {
          here.iov_base = (void *)key;
          here.iov_len = ksize;
          if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0))
            rc = bad_page(mp, "node[%zu] key wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here));
          prev = here;
        }
      }
      if (is_branch(mp)) {
        if ((mc->checking & z_updating) == 0 && i == 0 && unlikely(ksize != 0))
          rc = bad_page(mp, "branch-node[%zu] wrong 0-node key-length (%zu)\n", i, ksize);
        const pgno_t ref = node_pgno(node);
        if (unlikely(ref < MIN_PAGENO) || (unlikely(ref >= mc->txn->geo.first_unallocated) &&
                                           (unlikely(ref >= mc->txn->geo.now) || !(mc->checking & z_retiring))))
          rc = bad_page(mp, "branch-node[%zu] wrong pgno (%u)\n", i, ref);
        if (unlikely(node_flags(node)))
          rc = bad_page(mp, "branch-node[%zu] wrong flags (%u)\n", i, node_flags(node));
        continue;
      }

      switch (node_flags(node)) {
      default:
        rc = bad_page(mp, "invalid node[%zu] flags (%u)\n", i, node_flags(node));
        break;
      case N_BIG /* data on large-page */:
      case 0 /* usual */:
      case N_TREE /* sub-db */:
      case N_TREE | N_DUP /* dupsorted sub-tree */:
      case N_DUP /* short sub-page */:
        break;
      }

      const size_t dsize = node_ds(node);
      const char *const data = node_data(node);
      if (node_flags(node) & N_BIG) {
        if (unlikely(end_of_page < data + sizeof(pgno_t))) {
          rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "bigdata-pgno", i, nkeys, dsize,
                        data + dsize - end_of_page);
          continue;
        }
        if (unlikely(dsize <= v_clc.lmin || dsize > v_clc.lmax))
          rc = bad_page(mp, "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin,
                        v_clc.lmax);
        if (unlikely(node_size_len(node_ks(node), dsize) <= mc->txn->env->leaf_nodemax) &&
            mc->tree != &mc->txn->dbs[FREE_DBI])
          poor_page(mp, "too small data (%zu bytes) for bigdata-node", dsize);

        if ((mc->checking & z_retiring) == 0) {
          const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid);
          if (unlikely(lp.err != MDBX_SUCCESS))
            return lp.err;
          cASSERT0(mc, page_type(lp.page) == P_LARGE);
          const unsigned npages = largechunk_npages(env, dsize);
          if (unlikely(lp.page->pages != npages)) {
            if (lp.page->pages < npages)
              rc = bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize);
            else if (mc->tree != &mc->txn->dbs[FREE_DBI])
              poor_page(lp.page, "extra n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize);
          }
        }
        continue;
      }

      if (unlikely(end_of_page < data + dsize)) {
        rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "data", i, nkeys, dsize,
                      data + dsize - end_of_page);
        continue;
      }

      switch (node_flags(node)) {
      default:
        /* wrong, but already handled */
        continue;
      case 0 /* usual */:
        if (unlikely(dsize < v_clc.lmin || dsize > v_clc.lmax)) {
          rc = bad_page(mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin, v_clc.lmax);
          continue;
        }
        break;
      case N_TREE /* sub-db */:
        if (unlikely(dsize != sizeof(tree_t))) {
          rc = bad_page(mp, "invalid %s-db record size (%zu, expect %zu)\n", "named sub", dsize, sizeof(tree_t));
          continue;
        }
        break;
      case N_TREE | N_DUP /* dupsorted sub-tree */:
        if (unlikely(dsize != sizeof(tree_t))) {
          rc = bad_page(mp, "invalid %s-db record size (%zu, expect %zu)\n", "nested", dsize, sizeof(tree_t));
          continue;
        }
        break;
      case N_DUP /* short sub-page */:
        if (unlikely(dsize <= PAGEHDRSZ)) {
          rc = bad_page(mp, "invalid nested/sub-page record size (%zu, expect %u)\n", dsize, PAGEHDRSZ);
          continue;
        } else {
          const page_t *const sp = (page_t *)data;
          switch (sp->flags &
                  /* ignore legacy P_DIRTY flag */ ~P_LEGACY_DIRTY) {
          case P_LEAF | P_SUBP:
          case P_LEAF | P_DUPFIX | P_SUBP:
            break;
          default:
            rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n", sp->flags);
            continue;
          }

          const char *const end_of_subpage = data + dsize;
          const intptr_t nsubkeys = page_numkeys(sp);
          if (unlikely(nsubkeys == 0) && !(mc->checking & z_updating) && mc->tree->items)
            rc = bad_page(mp, "no keys on a %s-page\n", is_dupfix_leaf(sp) ? "leaf2-sub" : "leaf-sub");

          MDBX_val sub_here, sub_prev = {0, 0};
          for (int ii = 0; ii < nsubkeys; ii++) {
            if (is_dupfix_leaf(sp)) {
              /* DUPFIX pages have no entries[] or node headers */
              const size_t sub_ksize = sp->dupfix_ksize;
              const char *const sub_key = page_dupfix_ptr(sp, ii, mc->tree->dupfix_size);
              if (unlikely(end_of_subpage < sub_key + sub_ksize)) {
                rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage);
                continue;
              }

              if (unlikely(sub_ksize != v_clc.lmin)) {
                if (unlikely(sub_ksize < v_clc.lmin || sub_ksize > v_clc.lmax))
                  rc = bad_page(mp,
                                "nested-leaf2-key size (%zu) <> min/max "
                                "value-length (%zu/%zu)\n",
                                sub_ksize, v_clc.lmin, v_clc.lmax);
                else
                  v_clc.lmin = v_clc.lmax = sub_ksize;
              }
              if ((mc->checking & z_ignord) == 0) {
                sub_here.iov_base = (void *)sub_key;
                sub_here.iov_len = sub_ksize;
                if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0))
                  rc = bad_page(mp, "nested-leaf2-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev),
                                DVAL(&sub_here));
                sub_prev = sub_here;
              }
            } else {
              const node_t *const sub_node = page_node(sp, ii);
              const char *const sub_node_end = ptr_disp(sub_node, NODESIZE);
              if (unlikely(sub_node_end > end_of_subpage)) {
                rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n", end_of_subpage - sub_node_end);
                continue;
              }
              if (unlikely(node_flags(sub_node) != 0))
                rc = bad_page(mp, "nested-node invalid flags (%u)\n", node_flags(sub_node));

              const size_t sub_ksize = node_ks(sub_node);
              const char *const sub_key = node_key(sub_node);
              const size_t sub_dsize = node_ds(sub_node);
              /* char *sub_data = node_data(sub_node); */

              if (unlikely(sub_ksize < v_clc.lmin || sub_ksize > v_clc.lmax))
                rc = bad_page(mp,
                              "nested-node-key size (%zu) <> min/max "
                              "value-length (%zu/%zu)\n",
                              sub_ksize, v_clc.lmin, v_clc.lmax);
              if ((mc->checking & z_ignord) == 0) {
                sub_here.iov_base = (void *)sub_key;
                sub_here.iov_len = sub_ksize;
                if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0))
                  rc = bad_page(mp, "nested-node-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev),
                                DVAL(&sub_here));
                sub_prev = sub_here;
              }
              if (unlikely(sub_dsize != 0))
                rc = bad_page(mp, "nested-node non-empty data size (%zu)\n", sub_dsize);
              if (unlikely(end_of_subpage < sub_key + sub_ksize))
                rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage);
            }
          }
        }
        break;
      }
    }
  }
  return rc;
}

static __always_inline int check_page_header(const uint16_t ILL, const page_t *page, MDBX_txn *const txn,
                                             const txnid_t front) {
  if (unlikely(page->flags & ILL)) {
    if (ILL == P_ILL_BITS || (page->flags & P_ILL_BITS))
      return bad_page(page, "invalid page's flags (%u)\n", page->flags);
    else if (ILL & P_LARGE) {
      ASSERT((ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0);
      ASSERT(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX));
      return bad_page(page, "unexpected %s instead of %s (%u)\n", "large/overflow", "branch/leaf/leaf2", page->flags);
    } else if (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) {
      ASSERT((ILL & P_BRANCH) && (ILL & P_LEAF) && (ILL & P_DUPFIX));
      ASSERT(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX));
      return bad_page(page, "unexpected %s instead of %s (%u)\n", "branch/leaf/leaf2", "large/overflow", page->flags);
    } else {
      ASSERT(false);
    }
  }

  if (unlikely(page->txnid > front) && unlikely(page->txnid > txn->front_txnid || front < txn->txnid))
    return bad_page(page, "invalid page' txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n", page->txnid,
                    (front == txn->front_txnid && front != txn->txnid) ? "front-txn" : "parent-page", front);

  if (((ILL & P_LARGE) || !is_largepage(page)) && (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0) {
    /* Контроль четности page->upper тут либо приводит к ложным ошибкам,
     * либо слишком дорог по количеству операций. Заковырка в том, что upper
     * может быть нечетным на DUPFIX-страницах, при нечетном количестве
     * элементов нечетной длины. Поэтому четность page->upper здесь не
     * проверяется, но соответствующие полные проверки есть в page_check(). */
    if (unlikely(page->upper < page->lower || (page->lower & 1) || PAGEHDRSZ + page->upper > txn->env->ps))
      return bad_page(page, "invalid page' lower(%u)/upper(%u) with limit %zu\n", page->lower, page->upper,
                      page_space(txn->env));

  } else if ((ILL & P_LARGE) == 0) {
    const pgno_t npages = page->pages;
    if (unlikely(npages < 1) || unlikely(npages >= MAX_PAGENO / 2))
      return bad_page(page, "invalid n-pages (%u) for large-page\n", npages);
    if (unlikely(page->pgno + npages > txn->geo.first_unallocated))
      return bad_page(page, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", page->pgno + npages,
                      txn->geo.first_unallocated);
  } else {
    ASSERT(false);
  }
  return MDBX_SUCCESS;
}

__cold static __noinline pgr_t check_page_complete(const uint16_t ILL, page_t *page, const MDBX_cursor *const mc,
                                                   const txnid_t front) {
  pgr_t r = {page, check_page_header(ILL, page, mc->txn, front)};
  if (likely(r.err == MDBX_SUCCESS))
    r.err = page_check(mc, page);
  if (unlikely(r.err != MDBX_SUCCESS))
    mc->txn->flags |= MDBX_TXN_ERROR;
  return r;
}

__hot pgr_t page_get_unchecked(MDBX_txn *const txn, const pgno_t pgno, const txnid_t front) {
  tASSERT0(txn, front <= txn->front_txnid);

  pgr_t r;
  if (unlikely(pgno >= txn->geo.first_unallocated)) {
    ERROR("page #%" PRIaPGNO " beyond next-pgno", pgno);
    r.page = nullptr;
    r.err = MDBX_PAGE_NOTFOUND;
    return r;
  }

  tASSERT0(txn, ((txn->flags ^ txn->env->flags) & MDBX_WRITEMAP) == 0);
  r.page = pgno2page(txn->env, pgno);

#if MDBX_ENABLE_PGET_STAT
  txn->ops_pget += 1;
#endif /* MDBX_ENABLE_PGET_STAT */

  if ((txn->flags & (txn_ro_flat | MDBX_WRITEMAP)) == 0) {
    const MDBX_txn *spiller = txn;
    do {
      /* Spilled pages were dirtied in this txn and flushed because the dirty list got full. Bring this page back in
       * from the map (but don't unspill it here, * leave that unless page_touch happens again). */
      if (unlikely(spiller->flags & MDBX_TXN_SPILLS) && spill_search(spiller, pgno))
        break;

      if (spiller->flags & MDBX_TXN_DIRTY) {
        const size_t i = txn_dpl_search(spiller, pgno);
        cASSERT0(txn, (intptr_t)i > 0);
        if (spiller->wr.dirtylist->items[i].pgno == pgno) {
          r.page = spiller->wr.dirtylist->items[i].ptr;
          break;
        }
      }

      spiller = spiller->parent;
    } while (unlikely(spiller));
  }

  r.err = MDBX_SUCCESS;
  if (unlikely(r.page->pgno != pgno))
    r.err = bad_page(r.page, "pgno mismatch (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n", r.page->pgno, pgno);

  TRACE("page %u, %p, err %d", pgno, __Wpedantic_format_voidptr(r.page), r.err);
  return r;
}

static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_cursor *const mc, const pgno_t pgno,
                                             const txnid_t front) {
  MDBX_txn *const txn = mc->txn;
  cASSERT0(txn, front <= txn->front_txnid);

  pgr_t r = page_get_unchecked(mc->txn, pgno, front);
  if (likely(r.err == MDBX_SUCCESS)) {
    if (likely(mc->checking & z_pagecheck) == 0) {
#if MDBX_DISABLE_VALIDATION
      return r;
#else
      r.err = check_page_header(ILL, r.page, txn, front);
      if (likely(r.err == MDBX_SUCCESS))
        return r;
#endif /* MDBX_DISABLE_VALIDATION */
    } else
      return check_page_complete(ILL, r.page, mc, front);
  }

  txn->flags |= MDBX_TXN_ERROR;
  return r;
}

pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) {
  return page_get_inline(P_ILL_BITS, mc, pgno, front);
}

__hot pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) {
  return page_get_inline(P_ILL_BITS | P_LARGE, mc, pgno, front);
}

pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) {
  return page_get_inline(P_ILL_BITS | P_BRANCH | P_LEAF | P_DUPFIX, mc, pgno, front);
}

int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, mdbx_filehandle_t fd,
             bool check_coherence) {
  ctx->env = txn->env;
  ctx->ior = &txn->env->ioring;
  ctx->fd = fd;
  ctx->coherency_timestamp =
      (check_coherence || txn->env->lck->pgops.incoherence.weak) ? 0 : UINT64_MAX /* не выполнять сверку */;
  ctx->err = osal_ioring_prepare(ctx->ior, items, pgno_ceil2sp_bytes(txn->env, npages));
  if (likely(ctx->err == MDBX_SUCCESS)) {
#if MDBX_NEED_WRITTEN_RANGE
    ctx->flush_begin = MAX_PAGENO;
    ctx->flush_end = MIN_PAGENO;
#endif /* MDBX_NEED_WRITTEN_RANGE */
    osal_ioring_reset(ctx->ior);
  }
  return ctx->err;
}

static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes) {
  MDBX_env *const env = ctx->env;
  eASSERT0(env, (env->flags & MDBX_WRITEMAP) == 0);

  page_t *wp = (page_t *)data;
  eASSERT0(env, wp->pgno == bytes2pgno(env, offset));
  eASSERT0(env, bytes2pgno(env, bytes) >= (is_largepage(wp) ? wp->pages : 1u));
  eASSERT0(env, (wp->flags & P_ILL_BITS) == 0);

  if (likely(ctx->err == MDBX_SUCCESS)) {
    const page_t *const rp = ptr_disp(env->dxb_mmap.base, offset);
    VALGRIND_MAKE_MEM_DEFINED(rp, bytes);
    MDBX_ASAN_UNPOISON_MEMORY_REGION(rp, bytes);
    osal_flush_incoherent_mmap(rp, bytes, globals.sys_pagesize);
    /* check with timeout as the workaround
     * for https://libmdbx.dqdkfa.ru/dead-github/issues/269
     *
     * Проблема проявляется только при неупорядоченности: если записанная
     * последней мета-страница "обгоняет" ранее записанные, т.е. когда
     * записанное в файл позже становится видимым в отображении раньше,
     * чем записанное ранее.
     *
     * Исходно здесь всегда выполнялась полная сверка. Это давало полную
     * гарантию защиты от проявления проблемы, но порождало накладные расходы.
     * В некоторых сценариях наблюдалось снижение производительности до 10-15%,
     * а в синтетических тестах до 30%. Конечно никто не вникал в причины,
     * а просто останавливался на мнении "libmdbx не быстрее LMDB",
     * например: https://clck.ru/3386er
     *
     * Поэтому после серии экспериментов и тестов реализовано следующее:
     * 0. Посредством опции сборки MDBX_FORCE_CHECK_MMAP_COHERENCY=1
     *    можно включить полную сверку после записи.
     *    Остальные пункты являются взвешенным компромиссом между полной
     *    гарантией обнаружения проблемы и бесполезными затратами на системах
     *    без этого недостатка.
     * 1. При старте транзакций проверяется соответствие выбранной мета-страницы
     *    корневым страницам b-tree проверяется. Эта проверка показала себя
     *    достаточной без сверки после записи. При обнаружении "некогерентности"
     *    эти случаи подсчитываются, а при их ненулевом счетчике выполняется
     *    полная сверка. Таким образом, произойдет переключение в режим полной
     *    сверки, если показавшая себя достаточной проверка заметит проявление
     *    проблемы хоты-бы раз.
     * 2. Сверка не выполняется при фиксации транзакции, так как:
     *    - при наличии проблемы "не-когерентности" (при отложенном копировании
     *      или обновлении PTE, после возврата из write-syscall), проверка
     *      в этом процессе не гарантирует актуальность данных в другом
     *      процессе, который может запустить транзакцию сразу после коммита;
     *    - сверка только последнего блока позволяет почти восстановить
     *      производительность в больших транзакциях, но одновременно размывает
     *      уверенность в отсутствии сбоев, чем обесценивает всю затею;
     *    - после записи данных будет записана мета-страница, соответствие
     *      которой корневым страницам b-tree проверяется при старте
     *      транзакций, и только эта проверка показала себя достаточной;
     * 3. При спиллинге производится полная сверка записанных страниц. Тут был
     *    соблазн сверять не полностью, а например начало и конец каждого блока.
     *    Но при спиллинге возможна ситуация повторного вытеснения страниц, в
     *    том числе large/overflow. При этом возникает риск прочитать в текущей
     *    транзакции старую версию страницы, до повторной записи. В этом случае
     *    могут возникать крайне редкие невоспроизводимые ошибки. С учетом того
     *    что спиллинг выполняет крайне редко, решено отказаться от экономии
     *    в пользу надежности. */
#ifndef MDBX_FORCE_CHECK_MMAP_COHERENCY
#define MDBX_FORCE_CHECK_MMAP_COHERENCY 0
#endif /* MDBX_FORCE_CHECK_MMAP_COHERENCY */
    if ((MDBX_FORCE_CHECK_MMAP_COHERENCY || unlikely(ctx->coherency_timestamp != UINT64_MAX))) {
      VALGRIND_MAKE_MEM_DEFINED(wp, bytes);
      MDBX_ASAN_UNPOISON_MEMORY_REGION(wp, bytes);
      if (unlikely(memcmp(wp, rp, bytes))) {
        ctx->coherency_timestamp = 0;
        env->lck->pgops.incoherence.weak =
            (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1;
        WARNING("catch delayed/non-arrived page %" PRIaPGNO " %s", wp->pgno,
                "(workaround for incoherent flaw of unified page/buffer cache)");
        do
          if (coherency_timeout(&ctx->coherency_timestamp, wp->pgno, env) != MDBX_RESULT_TRUE) {
            ctx->err = MDBX_PROBLEM;
            break;
          }
        while (unlikely(memcmp(wp, rp, bytes)));
      }
    }
  }

  if (likely(bytes == env->ps))
    page_shadow_release(env, wp, 1);
  else {
    do {
      eASSERT0(env, wp->pgno == bytes2pgno(env, offset));
      eASSERT0(env, (wp->flags & P_ILL_BITS) == 0);
      size_t npages = is_largepage(wp) ? wp->pages : 1u;
      size_t chunk = pgno2bytes(env, npages);
      eASSERT0(env, bytes >= chunk);
      page_t *next = ptr_disp(wp, chunk);
      page_shadow_release(env, wp, npages);
      wp = next;
      offset += chunk;
      bytes -= chunk;
    } while (bytes);
  }
}

static void iov_complete(iov_ctx_t *ctx) {
  if ((ctx->env->flags & MDBX_WRITEMAP) == 0)
    osal_ioring_walk(ctx->ior, ctx, iov_callback4dirtypages);
  osal_ioring_reset(ctx->ior);
}

int iov_write(iov_ctx_t *ctx) {
  eASSERT0(ctx->env, !iov_empty(ctx));
  osal_ioring_write_result_t r = osal_ioring_write(ctx->ior, ctx->fd);
  if (MDBX_ENABLE_PGOP_STAT)
    ctx->env->lck->pgops.wops.weak += r.wops;
  ctx->err = r.err;
  if (unlikely(ctx->err != MDBX_SUCCESS))
    ERROR("Write error: %s", mdbx_strerror(ctx->err));
  iov_complete(ctx);
  return ctx->err;
}

int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) {
  MDBX_env *const env = txn->env;
  cASSERT0(txn, ctx->err == MDBX_SUCCESS);
  cASSERT0(txn, dp->pgno >= MIN_PAGENO && dp->pgno < txn->geo.first_unallocated);
  cASSERT0(txn, is_modifable(txn, dp));
  cASSERT0(txn, !(dp->flags & ~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE)));

  if (is_shadowed(txn, dp)) {
    cASSERT0(txn, !(txn->flags & MDBX_WRITEMAP));
    dp->txnid = txn->txnid;
    cASSERT0(txn, is_spilled(txn, dp));
#if MDBX_AVOID_MSYNC
  doit:;
#endif /* MDBX_AVOID_MSYNC */
    int err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages));
    if (unlikely(err != MDBX_SUCCESS)) {
      ctx->err = err;
      if (unlikely(err != MDBX_RESULT_TRUE)) {
        iov_complete(ctx);
        return err;
      }
      err = iov_write(ctx);
      cASSERT0(txn, iov_empty(ctx));
      if (likely(err == MDBX_SUCCESS)) {
        err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages));
        if (unlikely(err != MDBX_SUCCESS)) {
          iov_complete(ctx);
          return ctx->err = err;
        }
      }
      cASSERT0(txn, ctx->err == MDBX_SUCCESS);
    }
  } else {
    cASSERT0(txn, txn->flags & MDBX_WRITEMAP);
#if MDBX_AVOID_MSYNC
    goto doit;
#endif /* MDBX_AVOID_MSYNC */
  }

#if MDBX_NEED_WRITTEN_RANGE
  ctx->flush_begin = (ctx->flush_begin < dp->pgno) ? ctx->flush_begin : dp->pgno;
  ctx->flush_end = (ctx->flush_end > dp->pgno + (pgno_t)npages) ? ctx->flush_end : dp->pgno + (pgno_t)npages;
#endif /* MDBX_NEED_WRITTEN_RANGE */
  return MDBX_SUCCESS;
}

pgr_t page_new(MDBX_cursor *mc, const unsigned flags) {
  cASSERT0(mc, (flags & P_LARGE) == 0);
  pgr_t ret = gc_alloc_single(mc);
  if (unlikely(ret.err != MDBX_SUCCESS))
    return ret;

  DEBUG("db %zu allocated new page %" PRIaPGNO, cursor_dbi(mc), ret.page->pgno);
  ret.page->flags = (uint16_t)flags;
  cASSERT0(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
  cASSERT0(mc, mc->txn->flags & MDBX_TXN_DIRTY);
  if (MDBX_ENABLE_PGOP_STAT)
    mc->txn->env->lck->pgops.newly.weak += 1;
  STATIC_ASSERT(P_BRANCH == 1);
  const unsigned is_branch = flags & P_BRANCH;

  ret.page->lower = 0;
  ret.page->upper = (indx_t)(mc->txn->env->ps - PAGEHDRSZ);
  mc->tree->branch_pages += is_branch;
  mc->tree->leaf_pages += 1 - is_branch;
  if (unlikely(mc->flags & z_inner)) {
    tree_t *outer = cursor_outer_tree(mc);
    outer->branch_pages += is_branch;
    outer->leaf_pages += 1 - is_branch;
  }
  return ret;
}

pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
  pgr_t ret = likely(npages == 1) ? gc_alloc_single(mc) : gc_alloc_ex(mc, npages, ALLOC_DEFAULT);
  if (unlikely(ret.err != MDBX_SUCCESS))
    return ret;

  DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages);
  ret.page->flags = P_LARGE;
  cASSERT0(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
  cASSERT0(mc, mc->txn->flags & MDBX_TXN_DIRTY);
  if (MDBX_ENABLE_PGOP_STAT)
    mc->txn->env->lck->pgops.newly.weak += npages;
  mc->tree->large_pages += (pgno_t)npages;
  ret.page->pages = (pgno_t)npages;
  cASSERT0(mc, !(mc->flags & z_inner));
  return ret;
}

__hot void page_copy(page_t *const dst, const page_t *const src, const size_t size) {
  STATIC_ASSERT(UINT16_MAX > MDBX_MAX_PAGESIZE - PAGEHDRSZ);
  STATIC_ASSERT(MDBX_MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4);
  void *copy_dst = dst;
  const void *copy_src = src;
  size_t copy_len = size;
  if (src->flags & P_DUPFIX) {
    copy_len = PAGEHDRSZ + src->dupfix_ksize * page_numkeys(src);
    if (unlikely(copy_len > size))
      goto bailout;
  } else if ((src->flags & P_LARGE) == 0) {
    size_t upper = src->upper, lower = src->lower;
    intptr_t unused = upper - lower;
    /* If page isn't full, just copy the used portion. Adjust
     * alignment so memcpy may copy words instead of bytes. */
    if (unused > MDBX_CACHELINE_SIZE * 3) {
      lower = ceil_powerof2(lower + PAGEHDRSZ, sizeof(void *));
      upper = floor_powerof2(upper + PAGEHDRSZ, sizeof(void *));
      if (unlikely(upper > copy_len))
        goto bailout;
      memcpy(copy_dst, copy_src, lower);
      copy_dst = ptr_disp(copy_dst, upper);
      copy_src = ptr_disp(copy_src, upper);
      copy_len -= upper;
    }
  }
  memcpy(copy_dst, copy_src, copy_len);
  return;

bailout:
  if (src->flags & P_DUPFIX)
    bad_page(src, "%s addr %p, n-keys %zu, ksize %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src),
             page_numkeys(src), src->dupfix_ksize);
  else
    bad_page(src, "%s addr %p, upper %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src), src->upper);
  memset(dst, -1, size);
}

__cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp) {
  VERBOSE("unspill page %" PRIaPGNO, mp->pgno);
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0);
  cASSERT0(txn, is_spilled(txn, mp));
  const MDBX_txn *scan = txn;
  pgr_t ret;
  do {
    cASSERT0(txn, (scan->flags & MDBX_TXN_SPILLS) != 0);
    const size_t si = spill_search(scan, mp->pgno);
    if (!si)
      continue;
    const unsigned npages = is_largepage(mp) ? mp->pages : 1;
    ret.page = page_shadow_alloc(txn, npages);
    if (unlikely(!ret.page)) {
      ret.err = MDBX_ENOMEM;
      return ret;
    }
    page_copy(ret.page, mp, pgno2bytes(txn->env, npages));
    if (scan == txn) {
      /* If in current txn, this page is no longer spilled.
       * If it happens to be the last page, truncate the spill list.
       * Otherwise mark it as deleted by setting the LSB. */
      spill_remove(txn, si, npages);
    } /* otherwise, if belonging to a parent txn, the
       * page remains spilled until child commits */

    ret.err = page_dirty(txn, ret.page, npages);
    if (unlikely(ret.err != MDBX_SUCCESS))
      return ret;
    if (MDBX_ENABLE_PGOP_STAT)
      txn->env->lck->pgops.unspill.weak += npages;
    ret.page->flags |= (scan == txn) ? 0 : P_SPILLED;
    ret.err = MDBX_SUCCESS;
    return ret;
  } while (likely((scan = scan->parent) != nullptr && (scan->flags & MDBX_TXN_SPILLS) != 0));
  ERROR("Page %" PRIaPGNO " mod-txnid %" PRIaTXN " not found in the spill-list(s), current txn %" PRIaTXN
        " front %" PRIaTXN ", root txn %" PRIaTXN " front %" PRIaTXN,
        mp->pgno, mp->txnid, txn->txnid, txn->front_txnid, txn->env->basal_txn->txnid,
        txn->env->basal_txn->front_txnid);
  ret.err = MDBX_PROBLEM;
  ret.page = nullptr;
  return ret;
}

__hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) {
  cASSERT0(txn, is_modifable(txn, mp) && txn->wr.dirtylist);
  cASSERT0(txn, !is_largepage(mp) && !is_subpage(mp));
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

  const size_t n = txn_dpl_search(txn, mp->pgno);
  if (MDBX_AVOID_MSYNC && unlikely(txn->wr.dirtylist->items[n].pgno != mp->pgno)) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP));
    cASSERT0(txn, n > 0 && n <= txn->wr.dirtylist->length + 1);
    VERBOSE("unspill page %" PRIaPGNO, mp->pgno);
    if (MDBX_ENABLE_PGOP_STAT)
      txn->env->lck->pgops.unspill.weak += 1;
    return page_dirty(txn, (page_t *)mp, 1);
  }

  cASSERT0(txn, n > 0 && n <= txn->wr.dirtylist->length);
  cASSERT0(txn, txn->wr.dirtylist->items[n].pgno == mp->pgno && txn->wr.dirtylist->items[n].ptr == mp);
  if (!MDBX_AVOID_MSYNC || (txn->flags & MDBX_WRITEMAP) == 0) {
    size_t *const ptr = ptr_disp(txn->wr.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
    *ptr = txn->wr.dirtylru;
  }
  return MDBX_SUCCESS;
}

__hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp) {
  cASSERT0(txn, !is_modifable(txn, mp) && !is_largepage(mp));
  if (is_subpage(mp)) {
    ((page_t *)mp)->txnid = txn->front_txnid;
    return MDBX_SUCCESS;
  }

  int rc;
  page_t *np;
  if (is_frozen(txn, mp)) {
    /* CoW the page */
    const pgr_t par = gc_alloc_single(mc);
    rc = par.err;
    np = par.page;
    if (unlikely(rc != MDBX_SUCCESS)) {
      if (likely(mc->dbi_state != txn->dbi_state) || (rc != MDBX_MAP_FULL && rc != MDBX_BACKLOG_DEPLETED))
        goto fail;
      return rc;
    }

    rc = pnl_append(&txn->wr.retired_pages, mp->pgno);
    if (unlikely(rc != MDBX_SUCCESS))
      goto fail;

    const pgno_t pgno = np->pgno;
    DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
    cASSERT0(txn, mp->pgno != pgno);
    /* Update the parent page, if any, to point to the new page */
    if (likely(mc->top)) {
      page_t *parent = mc->pg[mc->top - 1];
      cASSERT0(mc, is_modifable(mc->txn, parent));
      node_t *node = page_node(parent, mc->ki[mc->top - 1]);
      node_set_pgno(node, pgno);
    } else {
      mc->tree->root = pgno;
    }

    if (MDBX_ENABLE_PGOP_STAT)
      txn->env->lck->pgops.cow.weak += 1;
    page_copy(np, mp, txn->env->ps);
    np->pgno = pgno;
    np->txnid = txn->front_txnid;
  } else if (is_spilled(txn, mp)) {
    pgr_t pur = page_unspill(txn, mp);
    np = pur.page;
    rc = pur.err;
    if (likely(rc == MDBX_SUCCESS)) {
      cASSERT0(txn, np != nullptr);
      goto done;
    }
    goto fail;
  } else {
    if (unlikely(!txn->parent)) {
      ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s "
            "page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
            " without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN,
            is_branch(mp) ? "branch" : "leaf", mp->pgno, mp->txnid, mc->txn->txnid, mc->txn->front_txnid);
      rc = MDBX_PROBLEM;
      goto fail;
    }

    DEBUG("clone db %d page %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno);
    cASSERT0(txn, txn->wr.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
    /* No - copy it */
    np = page_shadow_alloc(txn, 1);
    if (unlikely(!np)) {
      rc = MDBX_ENOMEM;
      goto fail;
    }
    page_copy(np, mp, txn->env->ps);

    /* insert a clone of parent's dirty page, so don't touch dirtyroom */
    rc = page_dirty(txn, np, 1);
    if (unlikely(rc != MDBX_SUCCESS))
      goto fail;

    if (MDBX_ENABLE_PGOP_STAT)
      txn->env->lck->pgops.clone.weak += 1;
  }

done:
  /* Adjust cursors pointing to mp */
  mc->pg[mc->top] = np;
#ifdef MDBX_EVENBUG20260405_FIX
  if (is_leaf(np) && inner_pointed(mc))
    /* 2026-04-05: Нашёлся занятный баг, унаследованный от LMDB -- Отсутствовало обновление вложенного dupsort-курсора.
     *
     * С одной стороны, отсутствие этого обновления не приводило к проблемам, так как во всех существовавших путях
     * выполнения вложенный курсор затем инициализировался заново.
     *
     * С другой стороны, такое обновление выполняется во всех остальных случаях, при условии что он связан с
     * dupsort-узлом на текущей/обновлённой странице, указывающем на sub-страницу, а не отдельное дерево. Причем во
     * многих вариантах развития событий, такая корректировка блокировалась явным присвоением `mc->pg[mc->top] = np`
     * выше.
     *
     * Однако, исправление "всё сломало", буквально из-за нарушения чётности багов.
     * В пути обработки put(append), позиция основного/внешнего курсора инкрементируется для последующей append-вставки,
     * но вложенный курсор при этом не сбрасывается. В результате, состояние курсорной пары становилось несогласованным.
     * При обновлении в отладочных сборках срабатывал assert, а в не-отладочных в dupsort-курсор записывалась чушь.
     *
     * По совокупности плюсов/минусов решено добавить это пояснение и оставить исправление под #ifdef, а обнаруженную
     * специфику учитывать при последующей разработке. */
    cursor_inner_refresh(mc, np, mc->ki[mc->top]);
#endif /* MDBX_EVENBUG20260405_FIX */
  MDBX_cursor *m2 = txn->cursors[cursor_dbi(mc)];
  if (mc->flags & z_inner) {
    for (; m2; m2 = m2->next) {
      MDBX_cursor *m3 = &m2->subcur->cursor;
      if (m3->top < mc->top)
        continue;
      if (m3->pg[mc->top] == mp)
        m3->pg[mc->top] = np;
    }
  } else {
    for (; m2; m2 = m2->next) {
      if (m2->top < mc->top)
        continue;
      if (m2->pg[mc->top] == mp) {
        m2->pg[mc->top] = np;
        if (is_leaf(np) && inner_pointed(m2))
          cursor_inner_refresh(m2, np, m2->ki[mc->top]);
      }
    }
  }
  return MDBX_SUCCESS;

fail:
  txn->flags |= MDBX_TXN_ERROR;
  return rc;
}

page_t *page_shadow_alloc(MDBX_txn *txn, size_t num) {
  MDBX_env *env = txn->env;
  page_t *np = env->shadow_reserve;
  size_t size = env->ps;
  if (likely(num == 1 && np)) {
    eASSERT0(env, env->shadow_reserve_len > 0);
    MDBX_ASAN_UNPOISON_MEMORY_REGION(np, size);
    VALGRIND_MEMPOOL_ALLOC(env, ptr_disp(np, -(ptrdiff_t)sizeof(size_t)), size + sizeof(size_t));
    VALGRIND_MAKE_MEM_DEFINED(&page_next(np), sizeof(page_t *));
    env->shadow_reserve = page_next(np);
    env->shadow_reserve_len -= 1;
  } else {
    size = pgno2bytes(env, num);
    void *const ptr = osal_malloc(size + sizeof(size_t));
    if (unlikely(!ptr)) {
      txn->flags |= MDBX_TXN_ERROR;
      return nullptr;
    }
    VALGRIND_MEMPOOL_ALLOC(env, ptr, size + sizeof(size_t));
    np = ptr_disp(ptr, sizeof(size_t));
  }

  if ((env->flags & MDBX_NOMEMINIT) == 0) {
    /* For a single page alloc, we init everything after the page header.
     * For multi-page, we init the final page; if the caller needed that
     * many pages they will be filling in at least up to the last page. */
    size_t skip = PAGEHDRSZ;
    if (num > 1)
      skip += pgno2bytes(env, num - 1);
    memset(ptr_disp(np, skip), 0, size - skip);
  }
#if MDBX_CHECKING > 0
  np->pgno = 0;
#endif
  VALGRIND_MAKE_MEM_UNDEFINED(np, size);
  np->flags = 0;
  np->pages = (pgno_t)num;
  return np;
}

void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages) {
  VALGRIND_MAKE_MEM_UNDEFINED(dp, pgno2bytes(env, npages));
  MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, pgno2bytes(env, npages));
  if (unlikely(env->flags & MDBX_PAGEPERTURB))
    memset(dp, -1, pgno2bytes(env, npages));
  if (likely(npages == 1 && env->shadow_reserve_len < env->options.dp_reserve_limit)) {
    MDBX_ASAN_POISON_MEMORY_REGION(dp, env->ps);
    MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *));
    page_next(dp) = env->shadow_reserve;
    VALGRIND_MEMPOOL_FREE(env, ptr_disp(dp, -(ptrdiff_t)sizeof(size_t)));
    env->shadow_reserve = dp;
    env->shadow_reserve_len += 1;
  } else {
    /* large pages just get freed directly */
    void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t));
    VALGRIND_MEMPOOL_FREE(env, ptr);
    osal_free(ptr);
  }
}

__cold static void page_kill(MDBX_txn *txn, page_t *mp, pgno_t pgno, size_t npages) {
  MDBX_env *const env = txn->env;
  DEBUG("kill %zu page(s) %" PRIaPGNO, npages, pgno);
  eASSERT0(env, pgno >= NUM_METAS && npages);
  if (!is_frozen(txn, mp)) {
    const size_t bytes = pgno2bytes(env, npages);
    memset(mp, -1, bytes);
    mp->pgno = pgno;
    if ((txn->flags & MDBX_WRITEMAP) == 0)
      osal_pwrite(env->lazy_fd, mp, bytes, pgno2bytes(env, pgno));
  } else {
    struct iovec iov[MDBX_AUXILARY_IOV_MAX];
    iov[0].iov_len = env->ps;
    iov[0].iov_base = ptr_disp(env->page_auxbuf, env->ps);
    size_t iov_off = pgno2bytes(env, pgno), n = 1;
    while (--npages) {
      iov[n] = iov[0];
      if (++n == MDBX_AUXILARY_IOV_MAX) {
        osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, iov_off);
        iov_off += pgno2bytes(env, MDBX_AUXILARY_IOV_MAX);
        n = 0;
      }
    }
    osal_pwritev(env->lazy_fd, iov, n, iov_off);
  }
}

static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) {
  /* TODO:
   *  1) при включенной "экономии последовательностей" проверить, что
   *     страница не примыкает к какой-либо из уже находящийся в reclaimed.
   *  2) стоит подумать над тем, чтобы при большом loose-списке отбрасывать
         половину в reclaimed. */
  return txn->wr.loose_count < txn->env->options.dp_loose_limit &&
         (!MDBX_ENABLE_REFUND ||
          /* skip pages near to the end in favor of compactification */
          txn->geo.first_unallocated > pgno + txn->env->options.dp_loose_limit ||
          txn->geo.first_unallocated <= txn->env->options.dp_loose_limit);
}

/* Retire, loosen or free a single page.
 *
 * For dirty pages, saves single pages to a list for future reuse in this same
 * txn. It has been pulled from the GC and already resides on the dirty list,
 * but has been deleted. Use these pages first before pulling again from the GC.
 *
 * If the page wasn't dirtied in this txn, just add it
 * to this txn's free list. */
int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */,
                   unsigned pageflags /* maybe unknown/zero */) {
  int rc;
  MDBX_txn *const txn = mc->txn;
  cASSERT0(txn, !mp || (mp->pgno == pgno && mp->flags == pageflags));

  /* During deleting entire subtrees, it is reasonable and possible to avoid
   * reading leaf pages, i.e. significantly reduce hard page-faults & IOPs:
   *  - mp is null, i.e. the page has not yet been read;
   *  - pagetype is known and the P_LEAF bit is set;
   *  - we can determine the page status via scanning the lists
   *    of dirty and spilled pages.
   *
   *  On the other hand, this could be suboptimal for WRITEMAP mode, since
   *  requires support the list of dirty pages and avoid explicit spilling.
   *  So for flexibility and avoid extra internal dependencies we just
   *  fallback to reading if dirty list was not allocated yet. */
  size_t di = 0, si = 0, npages = 1;
  enum page_status { unknown, frozen, spilled, shadowed, modifable } status = unknown;

  if (unlikely(!mp)) {
    if (CHECKS1_ENABLED() && pageflags) {
      pgr_t check;
      check = page_get_any(mc, pgno, txn->front_txnid);
      if (unlikely(check.err != MDBX_SUCCESS))
        return check.err;
      cASSERT0(txn, ((unsigned)check.page->flags & ~P_SPILLED) == (pageflags & ~P_FROZEN));
      cASSERT0(txn, !(pageflags & P_FROZEN) || is_frozen(txn, check.page));
    }
    if (pageflags & P_FROZEN) {
      status = frozen;
      if (CHECKS1_ENABLED()) {
        for (MDBX_txn *scan = txn; scan; scan = scan->parent) {
          tASSERT0(txn, !txn->wr.spilled.list || !spill_search(scan, pgno));
          tASSERT0(txn, !scan->wr.dirtylist || !debug_txn_dpl_find(scan, pgno));
        }
      }
      goto status_done;
    } else if (pageflags && txn->wr.dirtylist) {
      if ((di = txn_dpl_exist(txn, pgno)) != 0) {
        mp = txn->wr.dirtylist->items[di].ptr;
        cASSERT0(txn, is_modifable(txn, mp));
        status = modifable;
        goto status_done;
      }
      if ((si = spill_search(txn, pgno)) != 0) {
        status = spilled;
        goto status_done;
      }
      for (MDBX_txn *parent = txn->parent; parent; parent = parent->parent) {
        if (txn_dpl_exist(parent, pgno)) {
          status = shadowed;
          goto status_done;
        }
        if (spill_search(parent, pgno)) {
          status = spilled;
          goto status_done;
        }
      }
      status = frozen;
      goto status_done;
    }

    pgr_t pg = page_get_any(mc, pgno, txn->front_txnid);
    if (unlikely(pg.err != MDBX_SUCCESS))
      return pg.err;
    mp = pg.page;
    cASSERT0(txn, !pageflags || mp->flags == pageflags);
    pageflags = mp->flags;
  }

  if (is_frozen(txn, mp)) {
    status = frozen;
    cASSERT0(txn, !is_modifable(txn, mp));
    cASSERT0(txn, !is_spilled(txn, mp));
    cASSERT0(txn, !is_shadowed(txn, mp));
    tASSERT1(txn, !debug_txn_dpl_find(txn, pgno));
    tASSERT1(txn, !txn->wr.spilled.list || !spill_search(txn, pgno));
  } else if (is_modifable(txn, mp)) {
    status = modifable;
    if (txn->wr.dirtylist)
      di = txn_dpl_exist(txn, pgno);
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) || !is_spilled(txn, mp));
    tASSERT1(txn, !txn->wr.spilled.list || !spill_search(txn, pgno));
  } else if (is_shadowed(txn, mp)) {
    status = shadowed;
    tASSERT1(txn, !txn->wr.spilled.list || !spill_search(txn, pgno));
    tASSERT1(txn, !debug_txn_dpl_find(txn, pgno));
  } else {
    cASSERT0(txn, is_spilled(txn, mp));
    status = spilled;
    si = spill_search(txn, pgno);
    tASSERT1(txn, !debug_txn_dpl_find(txn, pgno));
  }

status_done:
  if (likely((pageflags & P_LARGE) == 0)) {
    STATIC_ASSERT(P_BRANCH == 1);
    const bool is_branch = pageflags & P_BRANCH;
    cASSERT0(mc, ((pageflags & P_LEAF) == 0) == is_branch);
    if (unlikely(mc->flags & z_inner)) {
      tree_t *outer = cursor_outer_tree(mc);
      cASSERT0(mc, !is_branch || outer->branch_pages > 0);
      outer->branch_pages -= is_branch;
      cASSERT0(mc, is_branch || outer->leaf_pages > 0);
      outer->leaf_pages -= 1 - is_branch;
    }
    cASSERT0(mc, !is_branch || mc->tree->branch_pages > 0);
    mc->tree->branch_pages -= is_branch;
    cASSERT0(mc, is_branch || mc->tree->leaf_pages > 0);
    mc->tree->leaf_pages -= 1 - is_branch;
  } else {
    npages = mp->pages;
    cASSERT0(mc, mc->tree->large_pages >= npages);
    mc->tree->large_pages -= (pgno_t)npages;
  }

  if (status == frozen) {
  retire:
    DEBUG("retire %zu page %" PRIaPGNO, npages, pgno);
    rc = pnl_append_span(&txn->wr.retired_pages, pgno, npages);
    tASSERT1(txn, txn_dpl_check(txn));
    return rc;
  }

  /* Возврат страниц в нераспределенный "хвост" БД.
   * Содержимое страниц не уничтожается, а для вложенных транзакций граница
   * нераспределенного "хвоста" БД сдвигается только при их коммите. */
  if (MDBX_ENABLE_REFUND && unlikely(pgno + npages == txn->geo.first_unallocated)) {
    const char *kind = nullptr;
    if (status == modifable) {
      /* Страница испачкана в этой транзакции, но до этого могла быть
       * аллоцирована, испачкана и пролита в одной из родительских транзакций.
       * Её МОЖНО вытолкнуть в нераспределенный хвост. */
      kind = "dirty";
      /* Remove from dirty list */
      page_wash(txn, di, mp, npages);
    } else if (si) {
      /* Страница пролита в этой транзакции, т.е. она аллоцирована
       * и запачкана в этой или одной из родительских транзакций.
       * Её МОЖНО вытолкнуть в нераспределенный хвост. */
      kind = "spilled";
      cASSERT0(txn, status == spilled);
      spill_remove(txn, si, npages);
    } else {
      /* Страница аллоцирована, запачкана и возможно пролита в одной
       * из родительских транзакций.
       * Её МОЖНО вытолкнуть в нераспределенный хвост. */
      kind = "parent's";
      if (CHECKS1_ENABLED() && mp) {
        kind = nullptr;
        for (MDBX_txn *parent = txn->parent; parent; parent = parent->parent) {
          if (spill_search(parent, pgno)) {
            kind = "parent-spilled";
            cASSERT0(txn, status == spilled);
            break;
          }
          if (mp == debug_txn_dpl_find(parent, pgno)) {
            kind = "parent-dirty";
            cASSERT0(txn, status == shadowed);
            break;
          }
        }
        cASSERT0(txn, kind != nullptr);
      }
      cASSERT0(txn, status == spilled || status == shadowed);
    }
    DEBUG("refunded %zu %s page %" PRIaPGNO, npages, kind, pgno);
    txn->geo.first_unallocated = pgno;
    txn_refund(txn);
    return MDBX_SUCCESS;
  }

  if (status == modifable) {
    /* Dirty page from this transaction */
    /* If suitable we can reuse it through loose list */
    if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->wr.dirtylist)) {
      DEBUG("loosen dirty page %" PRIaPGNO, pgno);
      if (MDBX_CHECKING > 1 || unlikely(txn->env->flags & MDBX_PAGEPERTURB))
        memset(page2payload(mp), -1, txn->env->ps - PAGEHDRSZ);
      mp->txnid = INVALID_TXNID;
      mp->flags = P_LOOSE;
      page_next(mp) = txn->wr.loose_pages;
      txn->wr.loose_pages = mp;
      txn->wr.loose_count++;
#if MDBX_ENABLE_REFUND
      txn->wr.loose_refund_wl = (pgno + 2 > txn->wr.loose_refund_wl) ? pgno + 2 : txn->wr.loose_refund_wl;
#endif /* MDBX_ENABLE_REFUND */
      VALGRIND_MAKE_MEM_NOACCESS(page2payload(mp), txn->env->ps - PAGEHDRSZ);
      MDBX_ASAN_POISON_MEMORY_REGION(page2payload(mp), txn->env->ps - PAGEHDRSZ);
      return MDBX_SUCCESS;
    }

#if MDBX_CHECKING < 1 && !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
    if (unlikely(txn->env->flags & MDBX_PAGEPERTURB))
#endif
    {
      /* Страница могла быть изменена в одной из родительских транзакций,
       * в том числе, позже выгружена и затем снова загружена и изменена.
       * В обоих случаях её нельзя затирать на диске и помечать недоступной
       * в asan и/или valgrind */
      for (MDBX_txn *parent = txn->parent; parent && (parent->flags & MDBX_TXN_SPILLS); parent = parent->parent) {
        if (spill_intersect(parent, pgno, npages))
          goto skip_invalidate;
        if (txn_dpl_intersect(parent, pgno, npages))
          goto skip_invalidate;
      }

#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
      if (MDBX_CHECKING > 1 || unlikely(txn->env->flags & MDBX_PAGEPERTURB))
#endif
        page_kill(txn, mp, pgno, npages);
      if ((txn->flags & MDBX_WRITEMAP) == 0) {
        VALGRIND_MAKE_MEM_NOACCESS(page2payload(pgno2page(txn->env, pgno)), pgno2bytes(txn->env, npages) - PAGEHDRSZ);
        MDBX_ASAN_POISON_MEMORY_REGION(page2payload(pgno2page(txn->env, pgno)),
                                       pgno2bytes(txn->env, npages) - PAGEHDRSZ);
      }
    }
  skip_invalidate:

    /* wash dirty page */
    page_wash(txn, di, mp, npages);

  reclaim:
    DEBUG("reclaim %zu %s page %" PRIaPGNO, npages, "dirty", pgno);
    rc = pnl_insert_span(&txn->wr.repnl, pgno, npages);
    if (MDBX_ENABLE_REFUND && unlikely(MDBX_PNL_MOST(txn->wr.repnl) == txn->geo.first_unallocated - 1)) {
      /* Refund suitable pages into "unallocated" space */
      txn_refund(txn);
    }
    tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
    tASSERT1(txn, txn_dpl_check(txn));
    return rc;
  }

  if (si) {
    /* Page ws spilled in this txn */
    spill_remove(txn, si, npages);
    /* Страница могла быть выделена и затем пролита в этой транзакции,
     * тогда её необходимо поместить в reclaimed-список.
     * Либо она могла быть выделена в одной из родительских транзакций и затем
     * пролита в этой транзакции, тогда её необходимо поместить в
     * retired-список для последующей фильтрации при коммите. */
    for (MDBX_txn *parent = txn->parent; parent; parent = parent->parent) {
      if (txn_dpl_exist(parent, pgno))
        goto retire;
    }
    /* Страница точно была выделена в этой транзакции
     * и теперь может быть использована повторно. */
    goto reclaim;
  }

  if (status == shadowed) {
    /* Dirty page MUST BE a clone from (one of) parent transaction(s). */
    if (CHECKS1_ENABLED()) {
      const page_t *parent_dp = nullptr;
      /* Check parent(s)'s dirty lists. */
      for (MDBX_txn *parent = txn->parent; parent && !parent_dp; parent = parent->parent) {
        tASSERT0(txn, !spill_search(parent, pgno));
        parent_dp = debug_txn_dpl_find(parent, pgno);
      }
      tASSERT0(txn, parent_dp && (!mp || parent_dp == mp));
    }
    /* Страница была выделена в родительской транзакции и теперь может быть
     * использована повторно, но только внутри этой транзакции, либо дочерних.
     */
    goto reclaim;
  }

  /* Страница может входить в доступный читателям MVCC-снимок, либо же она
   * могла быть выделена, а затем пролита в одной из родительских
   * транзакций. Поэтому пока помещаем её в retired-список, который будет
   * фильтроваться относительно dirty- и spilled-списков родительских
   * транзакций при коммите дочерних транзакций, либо же будет записан
   * в GC в неизменном виде. */
  goto retire;
}

__hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);
  mp->txnid = txn->front_txnid;
  if (!txn->wr.dirtylist) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
    txn->wr.writemap_dirty_npages += npages;
    cASSERT0(txn, txn->wr.spilled.list == nullptr);
    return MDBX_SUCCESS;
  }
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);

#if xMDBX_DEBUG_SPILLING == 2
  txn->env->debug_dirtied_act += 1;
  ENSURE(txn->env, txn->env->debug_dirtied_act < txn->env->debug_dirtied_est);
  ENSURE(txn->env, txn->wr.dirtyroom + txn->wr.loose_count > 0);
#endif /* xMDBX_DEBUG_SPILLING == 2 */

  int rc;
  if (unlikely(txn->wr.dirtyroom == 0)) {
    if (txn->wr.loose_count) {
      page_t *lp = txn->wr.loose_pages;
      DEBUG("purge-and-reclaim loose page %" PRIaPGNO, lp->pgno);
      rc = pnl_insert_span(&txn->wr.repnl, lp->pgno, 1);
      if (unlikely(rc != MDBX_SUCCESS))
        goto bailout;
      size_t di = txn_dpl_search(txn, lp->pgno);
      cASSERT0(txn, txn->wr.dirtylist->items[di].ptr == lp);
      txn_dpl_remove(txn, di);
      MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
      VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
      txn->wr.loose_pages = page_next(lp);
      txn->wr.loose_count--;
      txn->wr.dirtyroom++;
      if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP))
        page_shadow_release(txn->env, lp, 1);
      if (MDBX_ENABLE_REFUND && pnl_size(txn->wr.repnl) &&
          unlikely(MDBX_PNL_MOST(txn->wr.repnl) == txn->geo.first_unallocated - 1)) {
        /* Refund suitable pages into "unallocated" space */
        txn_refund(txn);
      }
    } else {
      ERROR("Dirtyroom is depleted, DPL length %zu", txn->wr.dirtylist->length);
      if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP))
        page_shadow_release(txn->env, mp, npages);
      return MDBX_TXN_FULL;
    }
  }

  rc = txn_dpl_append(txn, mp->pgno, mp, npages);
  if (unlikely(rc != MDBX_SUCCESS)) {
  bailout:
    txn->flags |= MDBX_TXN_ERROR;
    return rc;
  }
  txn->wr.dirtyroom--;
  tASSERT1(txn, txn_dpl_check(txn));
  return MDBX_SUCCESS;
}

void recalculate_subpage_thresholds(MDBX_env *env) {
  size_t whole = env->leaf_nodemax - NODESIZE;
  env->subpage_limit = (whole * env->options.subpage.limit + 32767) >> 16;
  whole = env->subpage_limit;
  env->subpage_reserve_limit = (whole * env->options.subpage.reserve_limit + 32767) >> 16;
  eASSERT0(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE);
  eASSERT0(env, env->subpage_limit >= env->subpage_reserve_limit);

  whole = env->leaf_nodemax;
  env->subpage_room_threshold = (whole * env->options.subpage.room_threshold + 32767) >> 16;
  env->subpage_reserve_prereq = (whole * env->options.subpage.reserve_prereq + 32767) >> 16;
  if (env->subpage_room_threshold + env->subpage_reserve_limit > (intptr_t)page_space(env))
    env->subpage_reserve_prereq = page_space(env);
  else if (env->subpage_reserve_prereq < env->subpage_room_threshold + env->subpage_reserve_limit)
    env->subpage_reserve_prereq = env->subpage_room_threshold + env->subpage_reserve_limit;
  eASSERT0(env, env->subpage_reserve_prereq >= env->subpage_room_threshold + env->subpage_reserve_limit);
}

size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len, size_t item_len) {
  eASSERT0(env, (subpage_len & 1) == 0);
  eASSERT0(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE);
  size_t reserve = 0;
  for (size_t n = 0; n < 5 && reserve + item_len <= env->subpage_reserve_limit &&
                     EVEN_CEIL(subpage_len + item_len) <= env->subpage_limit &&
                     host_page_room >= env->subpage_reserve_prereq + EVEN_CEIL(subpage_len + item_len);
       ++n) {
    subpage_len += item_len;
    reserve += item_len;
  }
  return reserve + (subpage_len & 1);
}

__hot int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata, const pgno_t newpgno,
                     const unsigned naf) {
  page_t *tmp_ki_copy = nullptr;
  MDBX_env *const env = mc->txn->env;
  DKBUF;

  page_t *const mp = mc->pg[mc->top];
  cASSERT0(mc, (mp->flags & P_ILL_BITS) == 0);

  DEBUG(">> splitting %s-page %" PRIaPGNO " and adding %zu+%zu [%s] at %i, nkeys %zi", is_leaf(mp) ? "leaf" : "branch",
        mp->pgno, newkey->iov_len, newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey), mc->ki[mc->top],
        page_numkeys(mp));

  /* Usually when splitting the root page, the cursor height is 1. But when called from tree_propagate_key(),
   * the cursor height may be greater because it walks up the stack while finding the branch slot to update. */
  intptr_t root_split_height = 0;
  int rc;
  if (mc->top == 0) {
    pgr_t npr = page_new(mc, P_BRANCH);
    rc = npr.err;
    if (unlikely(rc != MDBX_SUCCESS))
      goto done;
    /* shift current top to make room for new parent */
    cASSERT0(mc, mc->tree->height > 0);
#if MDBX_DEBUG > 0
    memset(mc->pg + 3, 0, sizeof(mc->pg) - sizeof(mc->pg[0]) * 3);
    memset(mc->ki + 3, -1, sizeof(mc->ki) - sizeof(mc->ki[0]) * 3);
#endif
    mc->pg[2] = mc->pg[1];
    mc->ki[2] = mc->ki[1];
    mc->pg[1] = mc->pg[0];
    mc->ki[1] = mc->ki[0];
    mc->pg[0] = npr.page;
    mc->ki[0] = 0;
    mc->tree->root = npr.page->pgno;
    DEBUG("root split! new root = %" PRIaPGNO, npr.page->pgno);
    root_split_height = mc->tree->height++;

    /* Add left (implicit) pointer. */
    rc = node_add_branch(mc, 0, nullptr, mp->pgno);
    if (unlikely(rc != MDBX_SUCCESS))
      goto done;

    mc->top = 1;
  } else {
    DEBUG("parent branch page is %" PRIaPGNO, mc->pg[mc->top - 1]->pgno);
  }

  const size_t nkeys = page_numkeys(mp);
  const size_t newindx = mc->ki[mc->top];
  STATIC_ASSERT(P_BRANCH == 1);
  const size_t minkeys = (mp->flags & P_BRANCH) + (size_t)1;
  cASSERT0(mc, nkeys + 1 >= minkeys * 2);
  const size_t leftmost_split = minkeys;
  const size_t rightmost_split = nkeys + 1 - minkeys;

  const size_t reserve_factor = env->options.split_reserve_dot16;
  size_t split_indx = (reserve_factor || env->options.prefer_waf_insteadof_balance) ? newindx : (nkeys + 1) >> 1;

  cASSERT0(mc, !is_branch(mp) || newindx > 0);
  MDBX_val sepkey = {nullptr, 0};
  if (!reserve_factor) {
    if (newindx == nkeys)
      split_indx = rightmost_split;
    else if (newindx <= minkeys) {
      split_indx = leftmost_split;
      /* It is reasonable and possible to split the page at the begin? */
      if (newindx == 0 && !(naf & MDBX_SPLIT_REPLACE)) {
        split_indx = 0;
        /* Checking for ability of splitting by the left-side insertion of a pure page with the new key. */
        for (intptr_t i = mc->top; --i >= 0;)
          if (mc->ki[i]) {
            sepkey = get_key(page_node(mc->pg[i], mc->ki[i]));
            ASSERT(mc->clc->k.cmp(newkey, &sepkey) >= 0);
            goto unable_pure_left;
          }

        /* Save the current first key which was omitted on the parent branch
         * page and should be updated if the new first entry will be added */
        if (is_dupfix_leaf(mp))
          sepkey = page_dupfix_key(mp, 0, mc->tree->dupfix_size);
        else
          sepkey = get_key(page_node(mp, 0));
        ASSERT(mc->clc->k.cmp(newkey, &sepkey) < 0);
        /* Avoiding rare complex cases of nested split the parent page(s) */
        if (page_room(mc->pg[mc->top - 1]) < branch_size(env, &sepkey)) {
          /* TODO: FIXME */
        unable_pure_left:
          split_indx = leftmost_split;
        }
      }
    }
  }
  const bool pure_right = !reserve_factor && split_indx == nkeys;
  const bool pure_left = !reserve_factor && split_indx == 0;

  /* Create a new sibling page. */
  pgr_t npr = page_new(mc, mp->flags);
  rc = npr.err;
  if (unlikely(rc != MDBX_SUCCESS))
    goto done;
  page_t *const sister = npr.page;
  sister->dupfix_ksize = mp->dupfix_ksize;
  DEBUG("new sibling: page %" PRIaPGNO, sister->pgno);
  cursor_couple_t couple;
  MDBX_cursor *const mn = cursor_clone_slightly(mc, &couple);
  mn->pg[mn->top] = sister;
  mn->ki[mn->top] = 0;
  intptr_t prev_top = mc->top - 1;
  mn->ki[prev_top] = mc->ki[prev_top] + 1;

  if (unlikely(pure_right)) {
    /* newindx == split_indx == nkeys */
    TRACE("no-split, but add new pure page at the %s", "right/after");
    cASSERT0(mc, newindx == nkeys && split_indx == nkeys && minkeys == 1);
    sepkey = *newkey;
  } else if (unlikely(pure_left)) {
    /* newindx == split_indx == 0 */
    TRACE("pure-left: no-split, but add new pure page at the %s", "left/before");
    cASSERT0(mc, newindx == 0 && split_indx == 0 && minkeys == 1);
    TRACE("pure-left: old-first-key is %s", DKEY_DEBUG(&sepkey));
  } else {
    split_indx = clamp_unsigned(split_indx, leftmost_split, rightmost_split);
    if (is_dupfix_leaf(sister)) {
      if (reserve_factor) {
        const size_t max_keys_per_page = nkeys;
        const size_t wanna_reserve = max_keys_per_page * reserve_factor >> 16;
        if (wanna_reserve /* + nkeys */ + 1 + wanna_reserve >= max_keys_per_page /* + max_keys_per_page */)
          split_indx = (nkeys + 1) >> 1;
        else {
          const size_t left_space = max_keys_per_page - split_indx - (newindx < split_indx);
          const size_t right_space = split_indx - (newindx >= split_indx);
          ASSERT(left_space + right_space + nkeys + 1 == max_keys_per_page * 2);
          if (wanna_reserve > left_space) {
            ASSERT(left_space < right_space);
            split_indx = max_unsigned(leftmost_split, max_keys_per_page - wanna_reserve -
                                                          (newindx < (max_keys_per_page - wanna_reserve)));
          } else if (wanna_reserve > right_space) {
            ASSERT(left_space > right_space);
            split_indx = min_unsigned(rightmost_split, wanna_reserve + (newindx >= wanna_reserve));
          }
        }
        ASSERT(split_indx >= leftmost_split && split_indx <= rightmost_split);
      }

      /* Move half of the keys to the right sibling */
      ASSERT(newindx == mc->ki[mc->top]);
      const intptr_t distance = newindx - split_indx;
      size_t ksize = mc->tree->dupfix_size;
      void *const split = page_dupfix_ptr(mp, split_indx, ksize);
      size_t rsize = (nkeys - split_indx) * ksize;
      size_t lsize = (nkeys - split_indx) * sizeof(indx_t);
      cASSERT0(mc, mp->lower >= lsize);
      mp->lower -= (indx_t)lsize;
      cASSERT0(mc, sister->lower + lsize <= UINT16_MAX);
      sister->lower += (indx_t)lsize;
      cASSERT0(mc, mp->upper + rsize - lsize <= UINT16_MAX);
      mp->upper += (indx_t)(rsize - lsize);
      cASSERT0(mc, sister->upper >= rsize - lsize);
      sister->upper -= (indx_t)(rsize - lsize);
      sepkey.iov_len = ksize;
      if (distance < 0) {
        cASSERT0(mc, ksize >= sizeof(indx_t));
        void *const ins = page_dupfix_ptr(mp, newindx, ksize);
        sepkey.iov_base = memcpy(sister->entries, split, rsize);
        memmove(ptr_disp(ins, ksize), ins, (split_indx - newindx) * ksize);
        memcpy(ins, newkey->iov_base, ksize);
        cASSERT0(mc, UINT16_MAX - mp->lower >= (int)sizeof(indx_t));
        mp->lower += sizeof(indx_t);
        cASSERT0(mc, mp->upper >= ksize - sizeof(indx_t));
        mp->upper -= (indx_t)(ksize - sizeof(indx_t));
        cASSERT0(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0);
      } else {
        sepkey.iov_base = (newindx != split_indx) ? split : newkey->iov_base;
        memcpy(sister->entries, split, distance * ksize);
        void *const ins = page_dupfix_ptr(sister, distance, ksize);
        memcpy(ins, newkey->iov_base, ksize);
        memcpy(ptr_disp(ins, ksize), ptr_disp(split, distance * ksize), rsize - distance * ksize);
        cASSERT0(mc, UINT16_MAX - sister->lower >= (int)sizeof(indx_t));
        sister->lower += sizeof(indx_t);
        cASSERT0(mc, sister->upper >= ksize - sizeof(indx_t));
        sister->upper -= (indx_t)(ksize - sizeof(indx_t));
        cASSERT0(mc, distance <= (int)UINT16_MAX);
        mc->ki[mc->top] = (indx_t)distance;
        cASSERT0(mc, (((ksize & page_numkeys(sister)) ^ sister->upper) & 1) == 0);
      }
    } else {
      /* grab a page to hold a temporary copy */
      tmp_ki_copy = page_shadow_alloc(mc->txn, 1);
      if (unlikely(!tmp_ki_copy)) {
        rc = MDBX_ENOMEM;
        goto done;
      }

      /* prepare to insert */
      size_t i = 0;
      while (i < newindx) {
        tmp_ki_copy->entries[i] = mp->entries[i];
        ++i;
      }
      tmp_ki_copy->entries[i] = (indx_t)-1;
      while (++i <= nkeys)
        tmp_ki_copy->entries[i] = mp->entries[i - 1];
      tmp_ki_copy->pgno = mp->pgno;
      tmp_ki_copy->flags = mp->flags;
      tmp_ki_copy->txnid = INVALID_TXNID;
      tmp_ki_copy->lower = 0;
      const size_t max_space = page_space(env);
      tmp_ki_copy->upper = (indx_t)max_space;

      const size_t newsize = is_leaf(mp) ? leaf_size(env, newkey, newdata) : branch_size(env, newkey);
      cASSERT0(mc, split_indx >= leftmost_split && split_indx <= rightmost_split);
      /* Добавляемый узел может не поместиться в страницу-половину вместе
       * с количественной половиной узлов из исходной страницы. В худшем случае,
       * в страницу-половину с добавляемым узлом могут попасть самые больше узлы
       * из исходной страницы, а другую половину только узлы с самыми короткими
       * ключами и с пустыми данными. Поэтому, чтобы найти подходящую границу
       * разреза требуется итерировать узлы подсчитывая объём.
       *
       * Однако, добавляемый узел точно поместится, если его размер не больше
       * чем место "освобождающееся" от заголовков узлов, которые переедут
       * в другую страницу-половину. Плюс, как минимум по одному байту
       * будет в каждом ключе, в худшем случае кроме одного, который может быть
       * нулевого размера. */
      const size_t dim_nodes = (newindx >= split_indx) ? split_indx : nkeys - split_indx;
      const size_t dim_bytes = (sizeof(indx_t) + NODESIZE + 1) * dim_nodes;
      if (reserve_factor || unlikely(newsize >= dim_bytes)) {
        const size_t reserve = reserve_factor ? max_space * reserve_factor >> 16 : 0;
        size_t left_size = 0;
        size_t right_size = newsize + page_used(env, mp);
        size_t best_split = split_indx;
        uint64_t best_skew = UINT64_MAX;
        for (i = 0; i < rightmost_split && left_size < max_space;) {
          size_t size = newsize;
          if (i != newindx) {
            node_t *node = ptr_disp(mp, tmp_ki_copy->entries[i] + PAGEHDRSZ);
            size = NODESIZE + node_ks(node) + sizeof(indx_t);
            if (is_leaf(mp))
              size += (node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node);
            size = EVEN_CEIL(size);
          }

          left_size += size;
          right_size -= size;
          if (++i >= leftmost_split && left_size <= max_space && right_size <= max_space) {
            const size_t split_skew = branchless_abs(split_indx - i);
            const size_t left_space = max_space - left_size;
            const size_t right_space = max_space - right_size;
            const uint64_t reserve_skew = ((reserve > left_space) ? reserve - left_space : 0) +
                                          ((reserve > right_space) ? reserve - right_space : 0);
            const uint64_t skew = (reserve_skew << 32) | split_skew;
            if (skew >= best_skew)
              break;
            best_split = i;
            best_skew = skew;
          }
        }
        cASSERT0(mc, best_split >= leftmost_split && best_split <= rightmost_split);
        split_indx = best_split;
      }

      sepkey = *newkey;
      if (split_indx != newindx) {
        node_t *node = ptr_disp(mp, tmp_ki_copy->entries[split_indx] + PAGEHDRSZ);
        sepkey.iov_len = node_ks(node);
        sepkey.iov_base = node_key(node);
      }
    }
  }
  DEBUG("separator is %zd [%s]", split_indx, DKEY_DEBUG(&sepkey));

  bool did_split_parent = false;
  /* Copy separator key to the parent. */
  if (unlikely(page_room(mn->pg[prev_top]) < branch_size(env, &sepkey))) {
    TRACE("need split parent branch-page for key %s", DKEY_DEBUG(&sepkey));
    cASSERT0(mc, page_numkeys(mn->pg[prev_top]) > 2);
    cASSERT0(mc, !pure_left);
    const int top = mc->top;
    const int height = mc->tree->height;
    mn->top -= 1;
    did_split_parent = true;
    couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
    mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
    rc = page_split(mn, &sepkey, nullptr, sister->pgno, 0);
    mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
    if (unlikely(rc != MDBX_SUCCESS))
      goto done;
    cASSERT0(mc, mc->top - top == mc->tree->height - height);
    if (CHECKS2_ENABLED())
      ENSURE_OBJ(mc, cursor_validate_updating(mc) == MDBX_SUCCESS);

    /* root split? */
    prev_top += mc->top - top;
    cASSERT0(mn, prev_top <= mn->top && prev_top <= mc->top);

    /* Right page might now have changed parent.
     * Check if left page also changed parent. */
    if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
      for (intptr_t i = 0; i < prev_top; i++) {
        mc->pg[i] = mn->pg[i];
        mc->ki[i] = mn->ki[i];
      }
      mc->pg[prev_top] = mn->pg[prev_top];
      if (mn->ki[prev_top]) {
        mc->ki[prev_top] = mn->ki[prev_top] - 1;
      } else {
        /* find right page's left sibling */
        mc->ki[prev_top] = mn->ki[prev_top];
        rc = cursor_sibling_left(mc);
        if (unlikely(rc != MDBX_SUCCESS)) {
          if (rc == MDBX_NOTFOUND) /* improper cursor_sibling() result */ {
            ERROR("unexpected %i error going left sibling", rc);
            rc = MDBX_PROBLEM;
          }
          goto done;
        }
      }
    }
  } else if (unlikely(pure_left)) {
    page_t *ptop_page = mc->pg[prev_top];
    TRACE("pure-left: adding to parent page %u node[%u] left-child page #%u key "
          "%s",
          ptop_page->pgno, mc->ki[prev_top], sister->pgno, DKEY(mc->ki[prev_top] ? newkey : nullptr));
    ASSERT(mc->top == prev_top + 1);
    mc->top = (uint8_t)prev_top;
    rc = node_add_branch(mc, mc->ki[prev_top], mc->ki[prev_top] ? newkey : nullptr, sister->pgno);
    cASSERT0(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1] && prev_top == mc->top);

    if (likely(rc == MDBX_SUCCESS) && mc->ki[prev_top] == 0) {
      node_t *node = page_node(mc->pg[prev_top], 1);
      TRACE("pure-left: update prev-first key on parent to %s", DKEY(&sepkey));
      cASSERT0(mc, node_ks(node) == 0 && node_pgno(node) == mp->pgno);
      cASSERT0(mc, mc->top == prev_top && mc->ki[prev_top] == 0);
      mc->ki[prev_top] = 1;
      rc = tree_propagate_key(mc, &sepkey);
      cASSERT0(mc, mc->top == prev_top && mc->ki[prev_top] == 1);
      cASSERT0(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1]);
      mc->ki[prev_top] = 0;
    } else {
      TRACE("pure-left: no-need-update prev-first key on parent %s", DKEY(&sepkey));
    }

    mc->top++;
    if (unlikely(rc != MDBX_SUCCESS))
      goto done;

    node_t *node = page_node(mc->pg[prev_top], mc->ki[prev_top] + (size_t)1);
    cASSERT0(mc, node_pgno(node) == mp->pgno && mc->pg[prev_top] == ptop_page);
  } else {
    mn->top -= 1;
    TRACE("add-to-parent the right-entry[%u] for new sibling-page", mn->ki[prev_top]);
    rc = node_add_branch(mn, mn->ki[prev_top], &sepkey, sister->pgno);
    mn->top += 1;
    if (unlikely(rc != MDBX_SUCCESS))
      goto done;
  }

  if (unlikely(pure_left | pure_right)) {
    mc->pg[mc->top] = sister;
    mc->ki[mc->top] = 0;
    cASSERT0(mc, newpgno == 0 || newpgno == P_INVALID);
    if (is_dupfix_leaf(sister)) {
      cASSERT0(mc, (naf & (N_BIG | N_TREE | N_DUP)) == 0);
      rc = node_add_dupfix(mc, 0, newkey);
    } else {
      cASSERT0(mc, is_leaf(sister));
      rc = node_add_leaf(mc, 0, newkey, newdata, naf);
    }
    if (unlikely(rc != MDBX_SUCCESS))
      goto done;

    if (pure_right) {
      for (intptr_t i = 0; i < mc->top; i++)
        mc->ki[i] = mn->ki[i];
    } else if (mc->ki[mc->top - 1] == 0) {
      for (intptr_t i = 2; i <= mc->top; ++i)
        if (mc->ki[mc->top - i]) {
          sepkey = get_key(page_node(mc->pg[mc->top - i], mc->ki[mc->top - i]));
          if (mc->clc->k.cmp(newkey, &sepkey) < 0) {
            mc->top -= (int8_t)i;
            DEBUG("pure-left: update new-first on parent [%i] page %u key %s", mc->ki[mc->top], mc->pg[mc->top]->pgno,
                  DKEY(newkey));
            rc = tree_propagate_key(mc, newkey);
            mc->top += (int8_t)i;
            if (unlikely(rc != MDBX_SUCCESS))
              goto done;
          }
          break;
        }
    }
  } else if (tmp_ki_copy) { /* !is_dupfix_leaf(mp) */
    /* Move nodes */
    mc->pg[mc->top] = sister;
    size_t n = 0, ii = split_indx;
    do {
      TRACE("i %zu, nkeys %zu => n %zu, rp #%u", ii, nkeys, n, sister->pgno);
      pgno_t pgno = 0;
      unsigned flags = naf;
      MDBX_val rkey, xdata;
      MDBX_val *rdata = nullptr;
      if (unlikely(ii == newindx)) {
        rkey = *newkey;
        if (is_leaf(mp))
          rdata = newdata;
        else
          pgno = newpgno;
        /* Update index for the new key. */
        mc->ki[mc->top] = (indx_t)n;
      } else {
        node_t *node = ptr_disp(mp, tmp_ki_copy->entries[ii] + PAGEHDRSZ);
        rkey.iov_base = node_key(node);
        rkey.iov_len = node_ks(node);
        if (is_leaf(mp)) {
          xdata.iov_base = node_data(node);
          xdata.iov_len = node_ds(node);
          rdata = &xdata;
        } else
          pgno = node_pgno(node);
        flags = node_flags(node);
      }

      if (is_leaf(sister)) {
        cASSERT0(mc, pgno == 0);
        cASSERT0(mc, rdata != nullptr);
        rc = node_add_leaf(mc, n, &rkey, rdata, flags);
      } else {
        cASSERT0(mc, 0 == (uint16_t)flags && is_branch(sister));
        /* First branch index doesn't need key data. */
        rc = node_add_branch(mc, n, n ? &rkey : nullptr, pgno);
      }
      if (unlikely(rc != MDBX_SUCCESS))
        goto done;

      ++n;
      if (++ii > nkeys) {
        ii = 0;
        n = 0;
        mc->pg[mc->top] = tmp_ki_copy;
        TRACE("switch to mp #%u", tmp_ki_copy->pgno);
      }
    } while (ii != split_indx);

    TRACE("ii %zu, nkeys %zu, n %zu, pgno #%u", ii, nkeys, n, mc->pg[mc->top]->pgno);

    n = page_numkeys(tmp_ki_copy);
    for (size_t i = 0; i < n; i++)
      mp->entries[i] = tmp_ki_copy->entries[i];
    mp->lower = tmp_ki_copy->lower;
    mp->upper = tmp_ki_copy->upper;
    memcpy(page_node(mp, n - 1), page_node(tmp_ki_copy, n - 1), env->ps - tmp_ki_copy->upper - PAGEHDRSZ);

    /* reset back to original page */
    if (newindx < split_indx) {
      mc->pg[mc->top] = mp;
    } else {
      mc->pg[mc->top] = sister;
      mc->ki[prev_top]++;
      /* Make sure ki is still valid. */
      if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
        for (intptr_t i = 0; i <= prev_top; i++) {
          mc->pg[i] = mn->pg[i];
          mc->ki[i] = mn->ki[i];
        }
      }
    }
  } else if (newindx >= split_indx) {
    mc->pg[mc->top] = sister;
    mc->ki[prev_top]++;
    /* Make sure ki is still valid. */
    if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
      for (intptr_t i = 0; i <= prev_top; i++) {
        mc->pg[i] = mn->pg[i];
        mc->ki[i] = mn->ki[i];
      }
    }
  }

  /* Adjust other cursors pointing to mp and/or to parent page */
  const size_t n = page_numkeys(mp);
  for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
    MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
    if (!is_pointed(m3) || m3 == mc)
      continue;
    if (root_split_height) {
      /* sub cursors may be on different DB */
      if (m3->pg[0] != mp)
        continue;
      /* root split */
      for (intptr_t k = root_split_height; k >= 0; k--) {
        m3->ki[k + 1] = m3->ki[k];
        m3->pg[k + 1] = m3->pg[k];
      }
      m3->ki[0] = m3->ki[0] >= n + pure_left;
      m3->pg[0] = mc->pg[0];
      m3->top += 1;
    }

    if (m3->top >= mc->top && m3->pg[mc->top] == mp && !pure_left) {
      if (m3->ki[mc->top] >= newindx)
        m3->ki[mc->top] += !(naf & MDBX_SPLIT_REPLACE);
      if (m3->ki[mc->top] >= n) {
        m3->pg[mc->top] = sister;
        cASSERT0(mc, m3->ki[mc->top] >= n);
        m3->ki[mc->top] -= (indx_t)n;
        for (intptr_t i = 0; i < mc->top; i++) {
          m3->ki[i] = mn->ki[i];
          m3->pg[i] = mn->pg[i];
        }
      }
    } else if (!did_split_parent && m3->top >= prev_top && m3->pg[prev_top] == mc->pg[prev_top] &&
               m3->ki[prev_top] >= mc->ki[prev_top]) {
      m3->ki[prev_top]++; /* also for the `pure-left` case */
    }
    if (inner_pointed(m3) && is_leaf(mp))
      cursor_inner_refresh(m3, m3->pg[mc->top], m3->ki[mc->top]);
  }
  TRACE("mp #%u left: %zd, sister #%u left: %zd", mp->pgno, page_room(mp), sister->pgno, page_room(sister));

done:
  if (tmp_ki_copy)
    page_shadow_release(env, tmp_ki_copy, 1);

  if (likely(rc == MDBX_SUCCESS)) {
    if (CHECKS2_ENABLED())
      ENSURE_OBJ(mc, cursor_validate_updating(mc) == MDBX_SUCCESS);
    if (unlikely(naf & MDBX_RESERVE)) {
      node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]);
      if (!(node_flags(node) & N_BIG))
        newdata->iov_base = node_data(node);
    }
    if (MDBX_ENABLE_PGOP_STAT)
      env->lck->pgops.split.weak += 1;
  } else {
    mc->txn->flags |= MDBX_TXN_ERROR;
    be_poor(mc);
  }

  DEBUG("<< mp #%u, rc %d", mp->pgno, rc);
  return rc;
}

pnl_t pnl_alloc(size_t size) {
  size_t bytes = pnl_size2bytes(size);
  pnl_t pnl = osal_malloc(bytes);
  if (likely(pnl)) {
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(pnl);
#endif /* osal_malloc_usable_size */
    pnl[0] = pnl_bytes2size(bytes);
    ASSERT(pnl[0] >= size);
    pnl += 1;
    *pnl = 0;
  }
  return pnl;
}

void pnl_free(pnl_t pnl) {
  if (likely(pnl))
    osal_free(pnl - 1);
}

pnl_t pnl_clone(const const_pnl_t src) {
  pnl_t pl = pnl_alloc(pnl_alloclen(src));
  if (likely(pl))
    memcpy(pl, src, MDBX_PNL_SIZEOF(src));
  return pl;
}

void pnl_shrink(pnl_t __restrict *__restrict ppnl) {
  ASSERT(pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) >= MDBX_PNL_INITIAL &&
         pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) < MDBX_PNL_INITIAL * 3 / 2);
  ASSERT(pnl_size(*ppnl) <= PAGELIST_LIMIT && pnl_alloclen(*ppnl) >= pnl_size(*ppnl));
  pnl_setsize(*ppnl, 0);
  if (unlikely(pnl_alloclen(*ppnl) >
               MDBX_PNL_INITIAL * (MDBX_PNL_PREALLOC_FOR_RADIXSORT ? 8 : 4) - MDBX_CACHELINE_SIZE / sizeof(pgno_t))) {
    size_t bytes = pnl_size2bytes(MDBX_PNL_INITIAL * 2);
    pnl_t pnl = osal_realloc(*ppnl - 1, bytes);
    if (likely(pnl)) {
#ifdef osal_malloc_usable_size
      bytes = osal_malloc_usable_size(pnl);
#endif /* osal_malloc_usable_size */
      *pnl = pnl_bytes2size(bytes);
      *ppnl = pnl + 1;
    }
  }
}

int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna) {
  if (unlikely(!*ppnl)) {
    *ppnl = pnl_alloc(wanna);
    return *ppnl ? MDBX_SUCCESS : MDBX_ENOMEM;
  }

  const size_t allocated = pnl_alloclen(*ppnl);
  ASSERT(pnl_size(*ppnl) <= PAGELIST_LIMIT && pnl_alloclen(*ppnl) >= pnl_size(*ppnl));
  if (unlikely(allocated >= wanna))
    return MDBX_SUCCESS;

  if (unlikely(wanna > /* paranoia */ PAGELIST_LIMIT)) {
    ERROR("PNL too long (%zu > %zu)", wanna, (size_t)PAGELIST_LIMIT);
    return MDBX_TXN_FULL;
  }

  const size_t size = (wanna + wanna - allocated < PAGELIST_LIMIT) ? wanna + wanna - allocated : PAGELIST_LIMIT;
  size_t bytes = pnl_size2bytes(size);
  pnl_t pnl = osal_realloc(*ppnl - 1, bytes);
  if (likely(pnl)) {
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(pnl);
#endif /* osal_malloc_usable_size */
    *pnl = pnl_bytes2size(bytes);
    ASSERT(*pnl >= wanna);
    *ppnl = pnl + 1;
    return MDBX_SUCCESS;
  }
  return MDBX_ENOMEM;
}

static __always_inline int __must_check_result pnl_append_stepped(unsigned step, __restrict pnl_t *ppnl, pgno_t pgno,
                                                                  size_t n) {
  ASSERT(n > 0);
  int rc = pnl_need(ppnl, n);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  const pnl_t pnl = *ppnl;
  if (likely(n == 1)) {
    pnl_append_prereserved(pnl, pgno);
    return MDBX_SUCCESS;
  }

#if MDBX_PNL_ASCENDING
  size_t w = pnl_size(pnl);
  do {
    pnl[++w] = pgno;
    pgno += step;
  } while (--n);
  pnl_setsize(pnl, w);
#else
  size_t w = pnl_size(pnl) + n;
  pnl_setsize(pnl, w);
  do {
    pnl[w--] = pgno;
    pgno += step;
  } while (--n);
#endif
  return MDBX_SUCCESS;
}

__hot int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
  return pnl_append_stepped(2, ppnl, pgno << 1, n);
}

__hot int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
  return pnl_append_stepped(1, ppnl, pgno, n);
}

int __must_check_result pnl_append_pnl(__restrict pnl_t *pdst, const const_pnl_t src) {
  if (likely(pnl_size(src) > 0)) {
    const size_t size = pnl_size(*pdst) + pnl_size(src);
    int err = pnl_reserve(pdst, size);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    memcpy(MDBX_PNL_END(*pdst), MDBX_PNL_BEGIN(src), pnl_size(src) * sizeof(pgno_t));
    pnl_setsize(*pdst, size);
  }
  return MDBX_SUCCESS;
}

__hot int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
  ASSERT(n > 0);
  int rc = pnl_need(ppnl, n);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  const pnl_t pnl = *ppnl;
  size_t r = pnl_size(pnl), w = r + n;
  pnl_setsize(pnl, w);
  while (r && MDBX_PNL_DISORDERED(pnl[r], pgno))
    pnl[w--] = pnl[r--];

  for (pgno_t fill = MDBX_PNL_ASCENDING ? pgno + n : pgno; w > r; --w)
    pnl[w] = MDBX_PNL_ASCENDING ? --fill : fill++;

  return MDBX_SUCCESS;
}

__hot __noinline bool pnl_check(const const_pnl_t pnl, const size_t limit) {
  ASSERT(limit >= MIN_PAGENO - MDBX_ENABLE_REFUND);
  if (likely(pnl_size(pnl))) {
    if (unlikely(pnl_size(pnl) > PAGELIST_LIMIT))
      return false;
    if (unlikely(MDBX_PNL_LEAST(pnl) < MIN_PAGENO))
      return false;
    if (unlikely(MDBX_PNL_MOST(pnl) >= limit))
      return false;

    if ((!MDBX_DISABLE_VALIDATION || CHECKS2_ENABLED()) && likely(pnl_size(pnl) > 1)) {
      const pgno_t *scan = MDBX_PNL_BEGIN(pnl);
      const pgno_t *const end = MDBX_PNL_END(pnl);
      pgno_t prev = *scan++;
      do {
        if (unlikely(!MDBX_PNL_ORDERED(prev, *scan)))
          return false;
        prev = *scan;
      } while (likely(++scan != end));
    }
  }
  return true;
}

static __always_inline void pnl_merge_inner(pgno_t *__restrict dst, const pgno_t *__restrict src_a,
                                            const pgno_t *__restrict src_b,
                                            const pgno_t *__restrict const src_b_detent) {
  do {
#if MDBX_HAVE_CMOV
    const bool flag = MDBX_PNL_ORDERED(*src_b, *src_a);
#if defined(__LCC__) || __CLANG_PREREQ(13, 0)
    // lcc 1.26: 13ШК (подготовка и первая итерация) + 7ШК (цикл), БЕЗ loop-mode
    // gcc>=7: cmp+jmp с возвратом в тело цикла (WTF?)
    // gcc<=6: cmov×3
    // clang<=12: cmov×3
    // clang>=13: cmov, set+add/sub
    *dst = flag ? *src_a-- : *src_b--;
#else
    // gcc: cmov, cmp+set+add/sub
    // clang<=5: cmov×2, set+add/sub
    // clang>=6: cmov, set+add/sub
    *dst = flag ? *src_a : *src_b;
    src_b += (ptrdiff_t)flag - 1;
    src_a -= flag;
#endif
    --dst;
#else  /* MDBX_HAVE_CMOV */
    while (MDBX_PNL_ORDERED(*src_b, *src_a))
      *dst-- = *src_a--;
    *dst-- = *src_b--;
#endif /* !MDBX_HAVE_CMOV */
  } while (likely(src_b > src_b_detent));
}

__hot size_t pnl_merge(pnl_t dst, const const_pnl_t src) {
  ASSERT(pnl_check_allocated(dst, MAX_PAGENO + 1));
  ASSERT(pnl_check(src, MAX_PAGENO + 1));
  const size_t src_len = pnl_size(src);
  const size_t dst_len = pnl_size(dst);
  size_t total = dst_len;
  ASSERT(pnl_alloclen(dst) >= total);
  if (likely(src_len > 0)) {
    total += src_len;
    if (MDBX_DEBUG < 1 && total < (MDBX_HAVE_CMOV ? 21 : 12))
      goto avoid_call_libc_for_short_cases;
    if (dst_len == 0 || MDBX_PNL_ORDERED(MDBX_PNL_LAST(dst), MDBX_PNL_FIRST(src)))
      memcpy(MDBX_PNL_END(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t));
    else if (MDBX_PNL_ORDERED(MDBX_PNL_LAST(src), MDBX_PNL_FIRST(dst))) {
      memmove(MDBX_PNL_BEGIN(dst) + src_len, MDBX_PNL_BEGIN(dst), dst_len * sizeof(pgno_t));
      memcpy(MDBX_PNL_BEGIN(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t));
    } else {
    avoid_call_libc_for_short_cases:
      dst[0] = /* the detent */ (MDBX_PNL_ASCENDING ? 0 : P_INVALID);
      pnl_merge_inner(dst + total, dst + dst_len, src + src_len, src);
    }
    pnl_setsize(dst, total);
  }
  ASSERT(pnl_check_allocated(dst, MAX_PAGENO + 1));
  return total;
}

#if MDBX_PNL_ASCENDING
#define MDBX_PNL_EXTRACT_KEY(ptr) (*(ptr))
#else
#define MDBX_PNL_EXTRACT_KEY(ptr) (P_INVALID - *(ptr))
#endif
RADIXSORT_IMPL(pgno, pgno_t, MDBX_PNL_EXTRACT_KEY, MDBX_PNL_PREALLOC_FOR_RADIXSORT, 0)

SORT_IMPL(pgno_sort, false, pgno_t, MDBX_PNL_ORDERED)

__hot __noinline void pnl_sort_nochk(pnl_t pnl) {
  if (likely(pnl_size(pnl) < MDBX_RADIXSORT_THRESHOLD) ||
      unlikely(!pgno_radixsort(&MDBX_PNL_FIRST(pnl), pnl_size(pnl))))
    pgno_sort(MDBX_PNL_BEGIN(pnl), MDBX_PNL_END(pnl));
}

SEARCH_IMPL(pgno_bsearch, pgno_t, pgno_t, MDBX_PNL_ORDERED)

__hot __noinline size_t pnl_search_nochk(const const_pnl_t pnl, pgno_t pgno) {
  const pgno_t *begin = MDBX_PNL_BEGIN(pnl);
  const pgno_t *it = pgno_bsearch(begin, pnl_size(pnl), pgno);
  const pgno_t *end = begin + pnl_size(pnl);
  ASSERT(it >= begin && it <= end);
  if (it != begin)
    ASSERT(MDBX_PNL_ORDERED(it[-1], pgno));
  if (it != end)
    ASSERT(!MDBX_PNL_ORDERED(it[0], pgno));
  return it - begin + 1;
}

size_t pnl_maxspan(const const_pnl_t pnl) {
  size_t len = pnl_size(pnl);
  if (len > 1) {
    size_t span = 1, left = len - span;
    const pgno_t *scan = MDBX_PNL_BEGIN(pnl);
    do {
      const bool contiguous = MDBX_PNL_CONTIGUOUS(*scan, scan[span], span);
      span += contiguous;
      scan += 1 - contiguous;
    } while (--left);
    len = span;
  }
  return len;
}

__hot pgno_t pnl_get_best_sequence(const pnl_t pnl, const size_t seq, const pgno_t defrag_detent) {
  size_t best_pos = 0;
  size_t best_extra = MAX_PAGENO;

#if MDBX_PNL_ASCENDING
#error "FIXME: Since 2026-04-01 alternatives to MDBX_PNL_ASCENDING = 0 are no longer supported."
#else
  size_t len = pnl_size(pnl);
  for (size_t span, i = len; i >= seq && pnl[i] <= defrag_detent - seq; i -= span) {
    span = 1;
    while (unlikely(MDBX_PNL_CONTIGUOUS(pnl[i - span], pnl[i], span)) && likely((intptr_t)(i - span) > 0))
      ++span;
    if (unlikely(span >= seq)) {
      size_t extra = span - seq;
      if (unlikely(extra < best_extra)) {
        best_pos = i;
        best_extra = extra;
        if (best_extra == 0)
          break;
      }
    }
  }

  pgno_t pgno = 0;
  if (best_pos) {
    pgno = pnl[best_pos];
    VERBOSE("seq %zu => %u", seq, pgno);
    ASSERT(pnl_contains_span(pnl, pgno, seq));
    ASSERT(pgno + seq <= defrag_detent);
    pnl_cut(pnl, best_pos - seq + 1, seq);
  }
#endif /* MDBX_PNL_ASCENDING */
  return pgno;
}

pgno_t pnl_crop_tail_sequence(const pnl_t pnl) {
  const size_t len = pnl_size(pnl);
  ASSERT(len > 0);
#if MDBX_PNL_ASCENDING
#error "FIXME: Since 2026-04-01 alternatives to MDBX_PNL_ASCENDING = 0 are no longer supported."
#else
  size_t span = 1;
  while (1 + span <= len && MDBX_PNL_CONTIGUOUS(pnl[1], pnl[1 + span], span))
    ++span;
  pnl_cut(pnl, 1, span);
#endif /* MDBX_PNL_ASCENDING */
  return span;
}

__hot void pnl_cut(pnl_t pnl, size_t pos, size_t span) {
  size_t len = pnl_size(pnl);
  ASSERT(pos > 0 && pos <= len && pos + span <= len + 1);
  if (likely(span > 0)) {
    pnl_setsize(pnl, len -= span);
    for (size_t i = pos; i <= len; ++i)
      pnl[i] = pnl[i + span];
  }
}

void pnl_sift(__restrict pnl_t pnl, const __restrict const_pnl_t fo) {
  if (pnl_size(pnl) && pnl_size(fo)) {
    const intptr_t fe = pnl_size(fo) + 1;
    const size_t len = pnl_size(pnl);
    size_t w, r = pnl_search(pnl, fo[1], MAX_PAGENO);
    for (intptr_t f = 1; r <= len;) { /* scan loop */
      ASSERT(f != fe);
      pgno_t fo_pgno = fo[f];
      pgno_t pl_pgno = pnl[r];
      if (likely(pl_pgno != fo_pgno)) {
        const bool cmp = MDBX_PNL_ORDERED(pl_pgno, fo_pgno);
        r += cmp;
        f += cmp ? 0 : 1;
        if (likely(f != fe))
          continue;
        return;
      }

      /* update loop */
      w = r;
    remove:
      ++r;
    next:
      ++f;
      if (unlikely(f == fe)) {
        while (r <= len)
          pnl[w++] = pnl[r++];
      } else {
        while (r <= len) {
          ASSERT(f != fe);
          fo_pgno = fo[f];
          pl_pgno = pnl[r];
          if (MDBX_PNL_ORDERED(pl_pgno, fo_pgno))
            pnl[w++] = pnl[r++];
          else if (MDBX_PNL_REVERSED(pl_pgno, fo_pgno))
            goto next;
          else
            goto remove;
        }
      }
      pnl_setsize(pnl, w - 1);
      return;
    }
  }
}

int pnl_cut_range(__restrict pnl_t pnl, __restrict pnl_t *const pdest, pgno_t range_begin, pgno_t range_end) {
  ASSERT(range_begin < range_end && pnl_size(pnl) && range_end > 0);
  const size_t from = pnl_search(pnl, MDBX_PNL_ASCENDING ? range_begin : range_end - 1, MAX_PAGENO);
  const size_t len = pnl_size(pnl);
  size_t to;
  for (to = from; to <= len; ++to) {
    pgno_t pgno = pnl[to];
    if (MDBX_PNL_ASCENDING ? (pgno >= range_end) : (pgno < range_begin))
      break;
    int err = pnl_append(pdest, pgno);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }
  if (from < to)
    pnl_cut(pnl, from, to - from);
  return MDBX_SUCCESS;
}

#if MDBX_ENABLE_REFUND
static void refund_reclaimed(MDBX_txn *txn) {
  /* Scanning in descend order */
  pgno_t first_unallocated = txn->geo.first_unallocated;
  const pnl_t pnl = txn->wr.repnl;
  cASSERT0(txn, pnl_size(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1);
#if MDBX_PNL_ASCENDING
  size_t i = pnl_size(pnl);
  cASSERT0(txn, pnl[i] == first_unallocated - 1);
  while (--first_unallocated, --i > 0 && pnl[i] == first_unallocated - 1)
    ;
  pnl_setsize(pnl, i);
#else
  size_t i = 1;
  cASSERT0(txn, pnl[i] == first_unallocated - 1);
  size_t len = pnl_size(pnl);
  while (--first_unallocated, ++i <= len && pnl[i] == first_unallocated - 1)
    ;
  pnl_setsize(pnl, len -= i - 1);
  for (size_t move = 0; move < len; ++move)
    pnl[1 + move] = pnl[i + move];
#endif
  VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, txn->geo.first_unallocated - first_unallocated,
          txn->geo.first_unallocated, first_unallocated);
  txn->geo.first_unallocated = first_unallocated;
  tASSERT1(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - 1));
}

static void refund_loose(MDBX_txn *txn) {
  cASSERT0(txn, txn->wr.loose_pages != nullptr);
  cASSERT0(txn, txn->wr.loose_count > 0);

  dpl_t *const dl = txn->wr.dirtylist;
  if (dl) {
    cASSERT0(txn, dl->length >= txn->wr.loose_count);
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
  } else {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
  }

  pgno_t onstack[MDBX_CACHELINE_SIZE * 8 / sizeof(pgno_t)];
  pnl_t suitable = onstack;

  if (!dl || dl->length - dl->sorted > txn->wr.loose_count) {
    /* Dirty list is useless since unsorted. */
    if (pnl_bytes2size(sizeof(onstack)) < txn->wr.loose_count) {
      suitable = pnl_alloc(txn->wr.loose_count);
      if (unlikely(!suitable))
        return /* this is not a reason for transaction fail */;
    }

    /* Collect loose-pages which may be refunded. */
    cASSERT0(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->wr.loose_count);
    pgno_t most = MIN_PAGENO;
    size_t w = 0;
    for (const page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
      cASSERT0(txn, lp->flags == P_LOOSE);
      cASSERT0(txn, txn->geo.first_unallocated > lp->pgno);
      if (likely(txn->geo.first_unallocated - txn->wr.loose_count <= lp->pgno)) {
        cASSERT0(txn, w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) : pnl_alloclen(suitable)));
        suitable[++w] = lp->pgno;
        most = (lp->pgno > most) ? lp->pgno : most;
      }
      MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
      VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
    }

    if (most + 1 == txn->geo.first_unallocated) {
      /* Sort suitable list and refund pages at the tail. */
      pnl_setsize(suitable, w);
      pnl_sort(suitable, MAX_PAGENO + 1);

      /* Scanning in descend order */
      const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1;
      const intptr_t begin = MDBX_PNL_ASCENDING ? pnl_size(suitable) : 1;
      const intptr_t end = MDBX_PNL_ASCENDING ? 0 : pnl_size(suitable) + 1;
      cASSERT0(txn, suitable[begin] >= suitable[end - step]);
      cASSERT0(txn, most == suitable[begin]);

      for (intptr_t i = begin + step; i != end; i += step) {
        if (suitable[i] != most - 1)
          break;
        most -= 1;
      }
      const size_t refunded = txn->geo.first_unallocated - most;
      DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, most, txn->geo.first_unallocated);
      txn->geo.first_unallocated = most;
      txn->wr.loose_count -= refunded;
      if (dl) {
        txn->wr.dirtyroom += refunded;
        dl->pages_including_loose -= refunded;
        ASSERT(txn->wr.dirtyroom <= txn->env->options.dp_limit);

        /* Filter-out dirty list */
        size_t r = 0;
        w = 0;
        if (dl->sorted) {
          do {
            if (dl->items[++r].pgno < most) {
              if (++w != r)
                dl->items[w] = dl->items[r];
            }
          } while (r < dl->sorted);
          dl->sorted = w;
        }
        while (r < dl->length) {
          if (dl->items[++r].pgno < most) {
            if (++w != r)
              dl->items[w] = dl->items[r];
          }
        }
        dpl_setlen(dl, w);
        cASSERT0(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
                          (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
      }
      goto unlink_loose;
    }
  } else {
    /* Dirtylist is mostly sorted, just refund loose pages at the end. */
    txn_dpl_sort(txn);
    cASSERT0(txn, dl->length < 2 || dl->items[1].pgno < dl->items[dl->length].pgno);
    cASSERT0(txn, dl->sorted == dl->length);

    /* Scan dirtylist tail-forward and cutoff suitable pages. */
    size_t n;
    for (n = dl->length; dl->items[n].pgno == txn->geo.first_unallocated - 1 && dl->items[n].ptr->flags == P_LOOSE;
         --n) {
      cASSERT0(txn, n > 0);
      page_t *dp = dl->items[n].ptr;
      DEBUG("refund-sorted page %" PRIaPGNO, dp->pgno);
      cASSERT0(txn, dp->pgno == dl->items[n].pgno);
      txn->geo.first_unallocated -= 1;
    }
    dpl_setlen(dl, n);

    if (dl->sorted != dl->length) {
      const size_t refunded = dl->sorted - dl->length;
      dl->sorted = dl->length;
      txn->wr.loose_count -= refunded;
      txn->wr.dirtyroom += refunded;
      dl->pages_including_loose -= refunded;
      cASSERT0(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
                        (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));

      /* Filter-out loose chain & dispose refunded pages. */
    unlink_loose:
      for (page_t *__restrict *__restrict link = &txn->wr.loose_pages; *link;) {
        page_t *dp = *link;
        cASSERT0(txn, dp->flags == P_LOOSE);
        MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *));
        VALGRIND_MAKE_MEM_DEFINED(&page_next(dp), sizeof(page_t *));
        if (txn->geo.first_unallocated > dp->pgno) {
          link = &page_next(dp);
        } else {
          *link = page_next(dp);
          if ((txn->flags & MDBX_WRITEMAP) == 0)
            page_shadow_release(txn->env, dp, 1);
        }
      }
    }
  }

  tASSERT1(txn, txn_dpl_check(txn));
  if (suitable != onstack)
    pnl_free(suitable);
  txn->wr.loose_refund_wl = txn->geo.first_unallocated;
}

bool txn_refund(MDBX_txn *txn) {
  const pgno_t before = txn->geo.first_unallocated;

  if (txn->wr.loose_pages && txn->wr.loose_refund_wl > txn->geo.first_unallocated)
    refund_loose(txn);

  while (true) {
    if (pnl_size(txn->wr.repnl) == 0 || MDBX_PNL_MOST(txn->wr.repnl) != txn->geo.first_unallocated - 1)
      break;

    refund_reclaimed(txn);
    if (!txn->wr.loose_pages || txn->wr.loose_refund_wl <= txn->geo.first_unallocated)
      break;

    const pgno_t memo = txn->geo.first_unallocated;
    refund_loose(txn);
    if (memo == txn->geo.first_unallocated)
      break;
  }

  if (before == txn->geo.first_unallocated)
    return false;

  if (txn->wr.spilled.list)
    /* Squash deleted pagenums if we refunded any */
    spill_purge(txn);

  return true;
}

#else /* MDBX_ENABLE_REFUND */

bool txn_refund(MDBX_txn *txn) {
  (void)txn;
  /* No online auto-compactification. */
  return false;
}

#endif /* MDBX_ENABLE_REFUND */

static inline size_t rkl_size2bytes(const size_t size) {
  ASSERT(size > 0 && size <= txl_max * 2);
  size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * size, txl_granulate * sizeof(txnid_t)) -
                 MDBX_ASSUME_MALLOC_OVERHEAD;
  return bytes;
}

static inline size_t rkl_bytes2size(const size_t bytes) {
  size_t size = bytes / sizeof(txnid_t);
  ASSERT(size > 0 && size <= txl_max * 2);
  return size;
}

void rkl_init(rkl_t *rkl) {
  rkl->list_limit = ARRAY_LENGTH(rkl->inplace);
  rkl->list = rkl->inplace;
  rkl_clear(rkl);
}

void rkl_clear(rkl_t *rkl) {
  rkl->solid_begin = UINT64_MAX;
  rkl->solid_end = 0;
  rkl->list_length = 0;
}

void rkl_destroy(rkl_t *rkl) {
  void *ptr = rkl->list;
  rkl->list = nullptr;
  if (ptr != rkl->inplace)
    osal_free(ptr);
}

static inline bool solid_empty(const rkl_t *rkl) { return !(rkl->solid_begin < rkl->solid_end); }

#define RKL_ORDERED(first, last) ((first) < (last))

SEARCH_IMPL(rkl_bsearch, txnid_t, txnid_t, RKL_ORDERED)

void rkl_destructive_move(rkl_t *src, rkl_t *dst) {
  ASSERT(rkl_check(src));
  dst->solid_begin = src->solid_begin;
  dst->solid_end = src->solid_end;
  dst->list_length = src->list_length;
  if (dst->list != dst->inplace)
    osal_free(dst->list);
  if (src->list != src->inplace) {
    dst->list = src->list;
    dst->list_limit = src->list_limit;
  } else {
    dst->list = dst->inplace;
    dst->list_limit = ARRAY_LENGTH(src->inplace);
    memcpy(dst->inplace, src->list, sizeof(dst->inplace));
  }
  rkl_init(src);
}

static int rkl_resize(rkl_t *rkl, size_t wanna_size) {
  ASSERT(wanna_size > rkl->list_length);
  ASSERT(rkl_check(rkl));
  STATIC_ASSERT(txl_max < INT_MAX / sizeof(txnid_t));
  if (unlikely(wanna_size > txl_max)) {
    ERROR("rkl too long (%zu >= %zu)", wanna_size, (size_t)txl_max);
    return MDBX_TXN_FULL;
  }
  if (unlikely(wanna_size < rkl->list_length)) {
    ERROR("unable shrink rkl to %zu since length is %u", wanna_size, rkl->list_length);
    return MDBX_PROBLEM;
  }

  if (unlikely(wanna_size <= ARRAY_LENGTH(rkl->inplace))) {
    if (rkl->list != rkl->inplace) {
      ASSERT(rkl->list_limit > ARRAY_LENGTH(rkl->inplace) && rkl->list_length <= ARRAY_LENGTH(rkl->inplace));
      memcpy(rkl->inplace, rkl->list, sizeof(rkl->inplace));
      rkl->list_limit = ARRAY_LENGTH(rkl->inplace);
      osal_free(rkl->list);
      rkl->list = rkl->inplace;
    } else {
      ASSERT(rkl->list_limit == ARRAY_LENGTH(rkl->inplace));
    }
    return MDBX_SUCCESS;
  }

  if (wanna_size != rkl->list_limit) {
    size_t bytes = rkl_size2bytes(wanna_size);
    void *ptr = (rkl->list == rkl->inplace) ? osal_malloc(bytes) : osal_realloc(rkl->list, bytes);
    if (unlikely(!ptr))
      return MDBX_ENOMEM;
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(ptr);
#endif /* osal_malloc_usable_size */
    rkl->list_limit = rkl_bytes2size(bytes);
    if (rkl->list == rkl->inplace)
      memcpy(ptr, rkl->inplace, sizeof(rkl->inplace));
    rkl->list = ptr;
  }
  return MDBX_SUCCESS;
}

int rkl_reserve(rkl_t *rkl, size_t wanna_size) {
  return (wanna_size > rkl->list_limit) ? rkl_resize(rkl, wanna_size) : MDBX_SUCCESS;
}

int rkl_copy(const rkl_t *src, rkl_t *dst) {
  ASSERT(rkl_check(src));
  rkl_init(dst);
  if (!rkl_empty(src)) {
    if (dst->list_limit < src->list_length) {
      int err = rkl_resize(dst, src->list_limit);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }
    memcpy(dst->list, src->list, sizeof(txnid_t) * src->list_length);
    dst->list_length = src->list_length;
    dst->solid_begin = src->solid_begin;
    dst->solid_end = src->solid_end;
  }
  return MDBX_SUCCESS;
}

size_t rkl_len(const rkl_t *rkl) { return rkl_empty(rkl) ? 0 : rkl->solid_end - rkl->solid_begin + rkl->list_length; }

__hot bool rkl_contain(const rkl_t *rkl, txnid_t id) {
  ASSERT(rkl_check(rkl));
  if (id >= rkl->solid_begin && id < rkl->solid_end)
    return true;
  if (rkl->list_length) {
    const txnid_t *it = rkl_bsearch(rkl->list, rkl->list_length, id);
    const txnid_t *const end = rkl->list + rkl->list_length;
    ASSERT(it >= rkl->list && it <= end);
    if (it != rkl->list)
      ASSERT(RKL_ORDERED(it[-1], id));
    if (it != end) {
      ASSERT(!RKL_ORDERED(it[0], id));
      return *it == id;
    }
  }
  return false;
}

__hot bool rkl_find(const rkl_t *rkl, const txnid_t id, rkl_iter_t *iter) {
  ASSERT(rkl_check(rkl));
  *iter = rkl_iterator(rkl, false);
  if (id >= rkl->solid_begin) {
    if (id < rkl->solid_end) {
      iter->pos = iter->solid_offset + (unsigned)(id - rkl->solid_begin);
      return true;
    }
    iter->pos = (unsigned)(rkl->solid_end - rkl->solid_begin);
  }
  if (rkl->list_length) {
    const txnid_t *it = rkl_bsearch(rkl->list, rkl->list_length, id);
    const txnid_t *const end = rkl->list + rkl->list_length;
    ASSERT(it >= rkl->list && it <= end);
    if (it != rkl->list)
      ASSERT(RKL_ORDERED(it[-1], id));
    iter->pos += (unsigned)(it - rkl->list);
    if (it != end) {
      ASSERT(!RKL_ORDERED(it[0], id));
      return *it == id;
    }
  }
  return false;
}

static inline txnid_t list_remove_first(rkl_t *rkl) {
  ASSERT(rkl->list_length > 0);
  const txnid_t first = rkl->list[0];
  if (--rkl->list_length) {
    /* TODO: Можно подумать о том, чтобы для избавления от memove() добавить headroom или вместо длины и
     * указателя на список использовать три поля: list_begin, list_end и list_buffer. */
    size_t i = 0;
    do
      rkl->list[i] = rkl->list[i + 1];
    while (++i < rkl->list_length);
  }
  return first;
}

static inline txnid_t after_cut(rkl_t *rkl, const txnid_t out) {
  if (rkl->list_length == 0 && rkl->solid_begin == rkl->solid_end) {
    rkl->solid_end = 0;
    rkl->solid_begin = UINT64_MAX;
  }
  return out;
}

static int extend_solid(rkl_t *rkl, txnid_t solid_begin, txnid_t solid_end, const txnid_t id) {
  if (rkl->list_length) {
    const txnid_t *i = rkl_bsearch(rkl->list, rkl->list_length, id);
    const txnid_t *const end = rkl->list + rkl->list_length;
    /* если начало или конец списка примыкает к непрерывному интервалу,
     * то переносим эти элементы из списка в непрерывный интервал */
    txnid_t *f = (txnid_t *)i;
    while (f > rkl->list && f[-1] >= solid_begin - 1) {
      f -= 1;
      solid_begin -= 1;
      if (unlikely(*f != solid_begin))
        return MDBX_RESULT_TRUE;
    }
    txnid_t *t = (txnid_t *)i;
    while (t < end && *t <= solid_end) {
      if (unlikely(*t != solid_end))
        return MDBX_RESULT_TRUE;
      solid_end += 1;
      t += 1;
    }
    if (f < t) {
      rkl->list_length -= t - f;
      while (t < end)
        *f++ = *t++;
    }
  }

  rkl->solid_begin = solid_begin;
  rkl->solid_end = solid_end;
  ASSERT(rkl_check(rkl));
  return MDBX_SUCCESS;
}

int rkl_push(rkl_t *rkl, const txnid_t id) {
  ASSERT(id >= MIN_TXNID && id < INVALID_TXNID);
  ASSERT(rkl_check(rkl));
  const bool known_continuous = false;

  if (rkl->solid_begin >= rkl->solid_end) {
    /* непрерывный интервал пуст */
    return extend_solid(rkl, id, id + 1, id);
  } else if (id < rkl->solid_begin) {
    if (known_continuous || id + 1 == rkl->solid_begin)
      /* id примыкает к solid_begin */
      return extend_solid(rkl, id, rkl->solid_end, id);
  } else if (id >= rkl->solid_end) {
    if (known_continuous || id == rkl->solid_end)
      /* id примыкает к solid_end */
      return extend_solid(rkl, rkl->solid_begin, id + 1, id);
  } else {
    /* id входит в интервал между solid_begin и solid_end, т.е. подан дубликат */
    return MDBX_RESULT_TRUE;
  }

  if (rkl->list_length == 1 && rkl->solid_end == rkl->solid_begin + 1 &&
      (rkl->list[0] == id + 1 || rkl->list[0] == id - 1)) {
    /* В списке один элемент и добавляемый id примыкает к нему, при этом в непрерывном интервале тоже один элемент.
     * Лучше поменять элементы списка и непрерывного интервала. */
    const txnid_t couple = (rkl->list[0] == id - 1) ? id - 1 : id;
    rkl->list[0] = rkl->solid_begin;
    rkl->solid_begin = couple;
    rkl->solid_end = couple + 2;
    ASSERT(rkl_check(rkl));
    return MDBX_SUCCESS;
  }

  if (unlikely(rkl->list_length == rkl->list_limit)) {
    /* удваиваем размер буфера если закончилось место */
    size_t x2 = (rkl->list_limit + 1) << 1;
    x2 = (x2 > 62) ? x2 : 62;
    x2 = (x2 < txl_max) ? x2 : txl_max;
    x2 = (x2 > rkl->list_length) ? x2 : rkl->list_length + 42;
    int err = rkl_resize(rkl, x2);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    ASSERT(rkl->list_limit > rkl->list_length);
  }

  size_t i = rkl->list_length;
  /* ищем место для вставки двигаясь от конца к началу списка, сразу переставляя/раздвигая элементы */
  while (i > 0) {
    if (RKL_ORDERED(id, rkl->list[i - 1])) {
      rkl->list[i] = rkl->list[i - 1];
      i -= 1;
      continue;
    }
    if (unlikely(id == rkl->list[i - 1])) {
      while (++i < rkl->list_length)
        rkl->list[i - 1] = rkl->list[i];
      return MDBX_RESULT_TRUE;
    }
    break;
  }

  rkl->list[i] = id;
  rkl->list_length++;
  ASSERT(rkl_check(rkl));

  /* После добавления id в списке могла образоваться длинная последовательность,
   * которую (возможно) стоит обменять с непрерывным интервалом. */
  if (rkl->list_length > ((MDBX_DEBUG > 0) ? 2 : 16) &&
      ((i > 0 && rkl->list[i - 1] == id - 1) || (i + 1 < rkl->list_length && rkl->list[i + 1] == id + 1))) {
    txnid_t new_solid_begin = id;
    size_t from = i;
    while (from > 0 && rkl->list[from - 1] == new_solid_begin - 1) {
      from -= 1;
      new_solid_begin -= 1;
    }
    txnid_t new_solid_end = id + 1;
    size_t to = i + 1;
    while (to < rkl->list_length && rkl->list[to] == new_solid_end) {
      to += 1;
      new_solid_end += 1;
    }

    const size_t new_solid_len = to - from;
    if (new_solid_len > 3) {
      const size_t old_solid_len = rkl->solid_end - rkl->solid_begin;
      if (new_solid_len > old_solid_len) {
        /* Новая непрерывная последовательность длиннее текущей.
         * Считаем обмен выгодным, если он дешевле пути развития событий с добавлением следующего элемента в список. */
        const size_t old_solid_pos = rkl_bsearch(rkl->list, rkl->list_length, rkl->solid_begin) - rkl->list;
        const size_t swap_cost =
            /* количество элементов списка после изымаемой из списка последовательности,
             * которые нужно переместить */
            rkl->list_length - to +
            /* количество элементов списка после позиции добавляемой в список последовательности,
             * которые нужно переместить */
            ((from > old_solid_pos) ? from - old_solid_pos : 0)
            /* количество элементов списка добавляемой последовательности, которые нужно добавить */
            + old_solid_len;
        /* количество элементов списка, которые нужно переместить для вставки еще-одного/следующего элемента */
        const size_t new_insert_cost = rkl->list_length - i;
        /* coverity[logical_vs_bitwise] */
        if (unlikely(swap_cost < new_insert_cost) || MDBX_DEBUG > 0) {
          /* Изымаемая последовательность длиннее добавляемой, поэтому:
           *  - список станет короче;
           *  - перемещать хвост нужно всегда к началу;
           *  - если начальные элементы потребуется раздвигать,
           *    то места хватит и остающиеся элементы в конце не будут перезаписаны. */
          size_t moved = 0;
          if (from > old_solid_pos) {
            /* добавляемая последовательность ближе к началу, нужно раздвинуть элементы в голове для вставки. */
            moved = from - old_solid_pos;
            do {
              from -= 1;
              rkl->list[from + old_solid_len] = rkl->list[from];
            } while (from > old_solid_pos);
          } else if (from + new_solid_len < old_solid_pos) {
            /* добавляемая последовательность дальше от начала,
             * перемещаем часть элементов из хвоста после изымаемой последовательности */
            do
              rkl->list[from++] = rkl->list[to++];
            while (from < old_solid_pos - new_solid_len);
          }

          /* вставляем последовательноть */
          i = 0;
          do
            rkl->list[from++] = rkl->solid_begin + i++;
          while (i != old_solid_len);

          /* сдвигаем оставшийся хвост */
          while (to < rkl->list_length)
            rkl->list[moved + from++] = rkl->list[to++];

          rkl->list_length = rkl->list_length - new_solid_len + old_solid_len;
          rkl->solid_begin = new_solid_begin;
          rkl->solid_end = new_solid_end;
          ASSERT(rkl_check(rkl));
        }
      }
    }
  }
  return MDBX_SUCCESS;
}

txnid_t rkl_pop(rkl_t *rkl, const bool highest_not_lowest) {
  ASSERT(rkl_check(rkl));

  if (rkl->list_length) {
    ASSERT(rkl->solid_begin <= rkl->solid_end);
    if (highest_not_lowest && (solid_empty(rkl) || rkl->solid_end < rkl->list[rkl->list_length - 1]))
      return after_cut(rkl, rkl->list[rkl->list_length -= 1]);
    if (!highest_not_lowest && (solid_empty(rkl) || rkl->solid_begin > rkl->list[0]))
      return after_cut(rkl, list_remove_first(rkl));
  }

  if (!solid_empty(rkl))
    return after_cut(rkl, highest_not_lowest ? --rkl->solid_end : rkl->solid_begin++);

  ASSERT(rkl_empty(rkl));
  return 0;
}

txnid_t rkl_lowest(const rkl_t *rkl) {
  if (rkl->list_length)
    return (solid_empty(rkl) || rkl->list[0] < rkl->solid_begin) ? rkl->list[0] : rkl->solid_begin;
  return !solid_empty(rkl) ? rkl->solid_begin : INVALID_TXNID;
}

txnid_t rkl_highest(const rkl_t *rkl) {
  if (rkl->list_length)
    return (solid_empty(rkl) || rkl->list[rkl->list_length - 1] >= rkl->solid_end) ? rkl->list[rkl->list_length - 1]
                                                                                   : rkl->solid_end - 1;
  return !solid_empty(rkl) ? rkl->solid_end - 1 : 0;
}

int rkl_merge(const rkl_t *src, rkl_t *dst, bool ignore_duplicates) {
  if (src->list_length) {
    size_t i = src->list_length;
    do {
      int err = rkl_push(dst, src->list[i - 1]);
      if (unlikely(err != MDBX_SUCCESS) && (!ignore_duplicates || err != MDBX_RESULT_TRUE))
        return err;
    } while (--i);
  }

  txnid_t id = src->solid_begin;
  while (id < src->solid_end) {
    int err = rkl_push(dst, id);
    if (unlikely(err != MDBX_SUCCESS) && (!ignore_duplicates || err != MDBX_RESULT_TRUE))
      return err;
    ++id;
  }
  return MDBX_SUCCESS;
}

int rkl_destructive_merge(rkl_t *src, rkl_t *dst, bool ignore_duplicates) {
  int err = rkl_merge(src, dst, ignore_duplicates);
  rkl_destroy(src);
  return err;
}

rkl_iter_t rkl_iterator(const rkl_t *rkl, const bool reverse) {
  rkl_iter_t iter = {.rkl = rkl, .pos = reverse ? rkl_len(rkl) : 0, .solid_offset = 0};
  if (!solid_empty(rkl) && rkl->list_length) {
    const txnid_t *it = rkl_bsearch(rkl->list, rkl->list_length, rkl->solid_begin);
    const txnid_t *const end = rkl->list + rkl->list_length;
    ASSERT(it >= rkl->list && it <= end && (it == end || *it > rkl->solid_begin));
    (void)end;
    iter.solid_offset = it - rkl->list;
  }
  return iter;
}

txnid_t rkl_turn(rkl_iter_t *iter, const bool reverse) {
  ASSERT((unsigned)reverse == (unsigned)!!reverse);
  size_t pos = iter->pos - reverse;
  if (unlikely(pos >= rkl_len(iter->rkl)))
    return 0;

  iter->pos = pos + !reverse;
  ASSERT(iter->pos <= rkl_len(iter->rkl));

  const size_t solid_len = iter->rkl->solid_end - iter->rkl->solid_begin;
  if (iter->rkl->list_length) {
    if (pos < iter->solid_offset)
      return iter->rkl->list[pos];
    else if (pos < iter->solid_offset + solid_len)
      return iter->rkl->solid_begin + pos - iter->solid_offset;
    else
      return iter->rkl->list[pos - solid_len];
  }

  ASSERT(pos < solid_len);
  return iter->rkl->solid_begin + pos;
}

size_t rkl_left(rkl_iter_t *iter, const bool reverse) {
  ASSERT(iter->pos <= rkl_len(iter->rkl));
  return reverse ? iter->pos : rkl_len(iter->rkl) - iter->pos;
}

#if 1
#define DEBUG_HOLE(hole)                                                                                               \
  do {                                                                                                                 \
  } while (0)
#else
#define DEBUG_HOLE(hole)                                                                                               \
  do {                                                                                                                 \
    printf("  return-%sward: %d, ", reverse ? "back" : "for", __LINE__);                                               \
    if (hole.begin == hole.end)                                                                                        \
      printf("empty-hole\n");                                                                                          \
    else if (hole.end - hole.begin == 1)                                                                               \
      printf("hole %" PRIaTXN "\n", hole.begin);                                                                       \
    else                                                                                                               \
      printf("hole %" PRIaTXN "-%" PRIaTXN "\n", hole.begin, hole.end - 1);                                            \
    fflush(nullptr);                                                                                                   \
  } while (0)
#endif

rkl_hole_t rkl_hole(rkl_iter_t *iter, const bool reverse) {
  ASSERT((unsigned)reverse == (unsigned)!!reverse);
  rkl_hole_t hole;
  const size_t len = rkl_len(iter->rkl);
  size_t pos = iter->pos;
  if (unlikely(pos >= len)) {
    if (len == 0) {
      hole.begin = 1;
      hole.end = MAX_TXNID;
      iter->pos = 0;
      DEBUG_HOLE(hole);
      return hole;
    } else if (pos == len && reverse) {
      /* шаг назад из позиции на конце rkl */
    } else if (reverse) {
      hole.begin = 1;
      hole.end = 1 /* rkl_lowest(iter->rkl); */;
      iter->pos = 0;
      DEBUG_HOLE(hole);
      return hole;
    } else {
      hole.begin = MAX_TXNID /* rkl_highest(iter->rkl) + 1 */;
      hole.end = MAX_TXNID;
      iter->pos = len;
      DEBUG_HOLE(hole);
      return hole;
    }
  }

  const size_t solid_len = iter->rkl->solid_end - iter->rkl->solid_begin;
  if (iter->rkl->list_length) {
    /* список элементов не пуст */
    txnid_t here, there;
    for (size_t next;; pos = next) {
      next = reverse ? pos - 1 : pos + 1;
      if (pos < iter->solid_offset) {
        /* текущая позиция перед непрерывным интервалом */
        here = iter->rkl->list[pos];
        if (next == iter->solid_offset) {
          /* в следующей позиции начинается непрерывный интерал (при поиске вперед) */
          ASSERT(!reverse);
          hole.begin = here + 1;
          hole.end = iter->rkl->solid_begin;
          next += solid_len;
          ASSERT(hole.begin < hole.end /* зазор обязан быть, иначе это ошибка не-слияния */);
          /* зазор между элементом списка перед сплошным интервалом и началом интервала */
          iter->pos = next - 1;
          DEBUG_HOLE(hole);
          return hole;
        }
        if (next >= len)
          /* уперлись в конец или начало rkl */
          break;
        /* следующая позиция также перед непрерывным интервалом */
        there = iter->rkl->list[next];
      } else if (pos >= iter->solid_offset + solid_len) {
        /* текущая позиция после непрерывного интервала */
        here = (pos < len) ? iter->rkl->list[pos - solid_len] : MAX_TXNID;
        if (next >= len)
          /* уперлись в конец или начало rkl */
          break;
        if (next == iter->solid_offset + solid_len - 1) {
          /* в следующей позиции конец непрерывного интервала (при поиске назад) */
          ASSERT(reverse);
          hole.begin = iter->rkl->solid_end;
          hole.end = here;
          pos = iter->solid_offset;
          ASSERT(hole.begin < hole.end /* зазор обязан быть, иначе это ошибка не-слияния */);
          /* зазор между элементом списка после сплошного интервала и концом интервала */
          iter->pos = pos;
          DEBUG_HOLE(hole);
          return hole;
        }
        /* следующая позиция также после непрерывного интервала */
        there = iter->rkl->list[next - solid_len];
      } else if (reverse) {
        /* текущая позиция внутри непрерывного интервала и поиск назад */
        next = iter->solid_offset - 1;
        here = iter->rkl->solid_begin;
        if (next >= len)
          /* нет элементов списка перед непрерывным интервалом */
          break;
        /* предыдущая позиция перед непрерывным интервалом */
        there = iter->rkl->list[next];
      } else {
        /* текущая позиция внутри непрерывного интервала и поиск вперед */
        next = iter->solid_offset + solid_len;
        here = iter->rkl->solid_end - 1;
        if (next >= len)
          /* нет элементов списка после непрерывного интервала */
          break;
        /* следующая позиция после непрерывного интервала */
        there = iter->rkl->list[next - solid_len];
      }

      hole.begin = (reverse ? there : here) + 1;
      hole.end = reverse ? here : there;
      if (hole.begin < hole.end) {
        /* есть зазор между текущей и следующей позицией */
        iter->pos = next;
        DEBUG_HOLE(hole);
        return hole;
      }
    }

    if (reverse) {
      /* уперлись в начало rkl, возвращаем зазор перед началом rkl */
      hole.begin = 1;
      hole.end = here;
      iter->pos = 0;
      DEBUG_HOLE(hole);
    } else {
      /* уперлись в конец rkl, возвращаем зазор после конца rkl */
      hole.begin = here + 1;
      hole.end = MAX_TXNID;
      iter->pos = len;
      DEBUG_HOLE(hole);
    }
    return hole;
  }

  /* список элементов пуст, но есть непрерывный интервал */
  iter->pos = reverse ? 0 : len;
  if (reverse && pos < len) {
    /* возвращаем зазор перед непрерывным интервалом */
    hole.begin = 1;
    hole.end = iter->rkl->solid_begin;
    DEBUG_HOLE(hole);
  } else {
    /* возвращаем зазор после непрерывного интервала */
    hole.begin = iter->rkl->solid_end;
    hole.end = MAX_TXNID;
    DEBUG_HOLE(hole);
  }
  return hole;
}

bool rkl_check(const rkl_t *rkl) {
  if (!rkl)
    return false;
  if (rkl->list == rkl->inplace && unlikely(rkl->list_limit != ARRAY_LENGTH(rkl->inplace)))
    return false;
  if (unlikely(rkl->list_limit < ARRAY_LENGTH(rkl->inplace)))
    return false;

  if (rkl_empty(rkl))
    return rkl->list_length == 0 && solid_empty(rkl);

  if (rkl->list_length) {
    for (size_t i = 1; i < rkl->list_length; ++i)
      if (unlikely(!RKL_ORDERED(rkl->list[i - 1], rkl->list[i])))
        return false;
    if (!solid_empty(rkl) && rkl->solid_begin - 1 <= rkl->list[rkl->list_length - 1] &&
        rkl->solid_end >= rkl->list[0]) {
      /* непрерывный интервал "плавает" внутри списка, т.е. находится между какими-то соседними значениями */
      const txnid_t *it = rkl_bsearch(rkl->list, rkl->list_length, rkl->solid_begin);
      const txnid_t *const end = rkl->list + rkl->list_length;
      if (it < rkl->list || it > end)
        return false;
      if (it > rkl->list && it[-1] >= rkl->solid_begin)
        return false;
      if (it < end && it[0] <= rkl->solid_end)
        return false;
    }
  }

  return true;
}

typedef struct rthc_entry {
  MDBX_env *env;
} rthc_entry_t;

#if MDBX_DEBUG > 0
#define RTHC_INITIAL_LIMIT 1
#else
#define RTHC_INITIAL_LIMIT 16
#endif

static unsigned rthc_count, rthc_limit = RTHC_INITIAL_LIMIT;
static rthc_entry_t rthc_table_static[RTHC_INITIAL_LIMIT];
static rthc_entry_t *rthc_table = rthc_table_static;

static int uniq_peek(const osal_mmap_t *pending, osal_mmap_t *scan) {
  int rc;
  uint64_t bait;
  lck_t *const pending_lck = pending->lck;
  lck_t *const scan_lck = scan->lck;
  if (pending_lck) {
    bait = atomic_load64(&pending_lck->bait_uniqueness, mo_AcquireRelease);
    rc = MDBX_SUCCESS;
  } else {
    bait = 0 /* hush MSVC warning */;
    rc = osal_msync(scan, sizeof(lck_t), MDBX_SYNC_DATA);
    if (rc == MDBX_SUCCESS)
      rc = osal_pread(pending->fd, &bait, sizeof(scan_lck->bait_uniqueness), offsetof(lck_t, bait_uniqueness));
  }
  if (likely(rc == MDBX_SUCCESS) && bait == atomic_load64(&scan_lck->bait_uniqueness, mo_AcquireRelease))
    rc = MDBX_RESULT_TRUE;

  TRACE("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d", pending_lck ? "mem" : "file", bait,
        (rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc);
  return rc;
}

static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan, uint64_t *abra) {
  if (*abra == 0) {
    const uintptr_t tid = osal_thread_self();
    uintptr_t uit = 0;
    memcpy(&uit, &tid, (sizeof(tid) < sizeof(uit)) ? sizeof(tid) : sizeof(uit));
    *abra = rrxmrrxmsx_0(osal_monotime() + UINT64_C(5873865991930747) * uit);
  }
  const uint64_t cadabra =
      rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)osal_getpid()) << 24 | *abra >> 40;
  lck_t *const scan_lck = scan->lck;
  atomic_store64(&scan_lck->bait_uniqueness, cadabra, mo_AcquireRelease);
  *abra = *abra * UINT64_C(6364136223846793005) + 1;
  return uniq_peek(pending, scan);
}

__cold int rthc_uniq_check(const osal_mmap_t *pending, MDBX_env **found) {
  *found = nullptr;
  uint64_t salt = 0;
  for (size_t i = 0; i < rthc_count; ++i) {
    MDBX_env *const scan = rthc_table[i].env;
    if (!scan->lck_mmap.lck || &scan->lck_mmap == pending)
      continue;
    int err = atomic_load64(&scan->lck_mmap.lck->bait_uniqueness, mo_AcquireRelease)
                  ? uniq_peek(pending, &scan->lck_mmap)
                  : uniq_poke(pending, &scan->lck_mmap, &salt);
    if (err == MDBX_ENODATA) {
      uint64_t length = 0;
      if (likely(osal_filesize(pending->fd, &length) == MDBX_SUCCESS && length == 0)) {
        /* LY: skip checking since LCK-file is empty, i.e. just created. */
        DEBUG("%s", "unique (new/empty lck)");
        return MDBX_SUCCESS;
      }
    }
    if (err == MDBX_RESULT_TRUE)
      err = uniq_poke(pending, &scan->lck_mmap, &salt);
    if (err == MDBX_RESULT_TRUE) {
      (void)osal_msync(&scan->lck_mmap, sizeof(lck_t), MDBX_SYNC_KICK);
      err = uniq_poke(pending, &scan->lck_mmap, &salt);
    }
    if (err == MDBX_RESULT_TRUE) {
      err = uniq_poke(pending, &scan->lck_mmap, &salt);
      *found = scan;
      DEBUG("found %p", __Wpedantic_format_voidptr(*found));
      return MDBX_SUCCESS;
    }
    if (unlikely(err != MDBX_SUCCESS)) {
      DEBUG("failed rc %d", err);
      return err;
    }
  }

  DEBUG("%s", "unique");
  return MDBX_SUCCESS;
}

//------------------------------------------------------------------------------

#if defined(_WIN32) || defined(_WIN64)
static CRITICAL_SECTION rthc_critical_section;
#else

static pthread_mutex_t rthc_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t rthc_cond = PTHREAD_COND_INITIALIZER;
static osal_thread_key_t rthc_key;
static mdbx_atomic_uint32_t rthc_pending;

static inline uint64_t rthc_signature(const void *addr, uint8_t kind) {
  uint64_t salt = osal_thread_self() * UINT64_C(0xA2F0EEC059629A17) ^ UINT64_C(0x01E07C6FDB596497) * (uintptr_t)(addr);
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  return salt << 8 | kind;
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
  return (uint64_t)kind << 56 | salt >> 8;
#else
#error "FIXME: Unsupported byte order"
#endif /* __BYTE_ORDER__ */
}

#define MDBX_THREAD_RTHC_REGISTERED(addr) rthc_signature(addr, 0x0D)
#define MDBX_THREAD_RTHC_COUNTED(addr) rthc_signature(addr, 0xC0)
static __thread uint64_t rthc_thread_state
#if __has_attribute(tls_model) && (defined(__PIC__) || defined(__pic__) || MDBX_BUILD_SHARED_LIBRARY)
    __attribute__((tls_model("local-dynamic")))
#endif
    ;

#if defined(__APPLE__) && defined(__SANITIZE_ADDRESS__) && !defined(MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS)
/* Avoid ASAN-trap due the target TLS-variable feed by Darwin's tlv_free() */
#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((__no_sanitize_address__, __noinline__))
#else
#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS inline
#endif

MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_read(const void *rthc) { return *(volatile uint64_t *)rthc; }

MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_compare_and_clean(const void *rthc, const uint64_t signature) {
#if MDBX_64BIT_CAS
  return atomic_cas64((mdbx_atomic_uint64_t *)rthc, signature, 0);
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)signature, 0);
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
  return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)(signature >> 32), 0);
#else
#error "FIXME: Unsupported byte order"
#endif
}

static inline int rthc_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) {
#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL
#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL) || defined(HAVE___CXA_THREAD_ATEXIT_IMPL) ||                         \
    __GLIBC_PREREQ(2, 18) || defined(BIONIC)
#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 1
#else
#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 0
#endif
#endif /* MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL */

#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT
#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT) || defined(HAVE___CXA_THREAD_ATEXIT)
#define MDBX_HAVE_CXA_THREAD_ATEXIT 1
#elif !MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && (defined(__linux__) || defined(__gnu_linux__))
#define MDBX_HAVE_CXA_THREAD_ATEXIT 1
#else
#define MDBX_HAVE_CXA_THREAD_ATEXIT 0
#endif
#endif /* MDBX_HAVE_CXA_THREAD_ATEXIT */

  int rc = MDBX_ENOSYS;
#if MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && !MDBX_HAVE_CXA_THREAD_ATEXIT
#define __cxa_thread_atexit __cxa_thread_atexit_impl
#endif
#if MDBX_HAVE_CXA_THREAD_ATEXIT || defined(__cxa_thread_atexit)
  extern int __cxa_thread_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) MDBX_WEAK_IMPORT_ATTRIBUTE;
  if (&__cxa_thread_atexit)
    rc = __cxa_thread_atexit(dtor, obj, dso_symbol);
#elif defined(__APPLE__) || defined(_DARWIN_C_SOURCE)
  extern void _tlv_atexit(void (*termfunc)(void *objAddr), void *objAddr) MDBX_WEAK_IMPORT_ATTRIBUTE;
  if (&_tlv_atexit) {
    (void)dso_symbol;
    _tlv_atexit(dtor, obj);
    rc = 0;
  }
#else
  (void)dtor;
  (void)obj;
  (void)dso_symbol;
#endif
  return rc;
}

__cold void workaround_glibc_bug21031(void) {
  /* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031
   *
   * Due race between pthread_key_delete() and __nptl_deallocate_tsd()
   * The destructor(s) of thread-local-storage object(s) may be running
   * in another thread(s) and be blocked or not finished yet.
   * In such case we get a SEGFAULT after unload this library DSO.
   *
   * So just by yielding a few timeslices we give a chance
   * to such destructor(s) for completion and avoids segfault. */
  sched_yield();
  sched_yield();
  sched_yield();
}
#endif /* !Windows */

void rthc_lock(void) {
#if defined(_WIN32) || defined(_WIN64)
  EnterCriticalSection(&rthc_critical_section);
#else
  ENSURE(osal_pthread_mutex_lock(&rthc_mutex) == 0);
#endif
}

void rthc_unlock(void) {
#if defined(_WIN32) || defined(_WIN64)
  LeaveCriticalSection(&rthc_critical_section);
#else
  ENSURE(pthread_mutex_unlock(&rthc_mutex) == 0);
#endif
}

static inline int thread_key_create(osal_thread_key_t *key) {
  int rc;
#if defined(_WIN32) || defined(_WIN64)
  *key = TlsAlloc();
  rc = (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError();
#else
  rc = pthread_key_create(key, nullptr);
#endif
  TRACE("&key = %p, value %" PRIuPTR ", rc %d", __Wpedantic_format_voidptr(key), (uintptr_t)*key, rc);
  return rc;
}

void thread_rthc_set(osal_thread_key_t key, const void *value) {
#if defined(_WIN32) || defined(_WIN64)
  ENSURE(TlsSetValue(key, (void *)value));
#else
  const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(&rthc_thread_state);
  const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(&rthc_thread_state);
  if (value && unlikely(rthc_thread_state != sign_registered && rthc_thread_state != sign_counted)) {
    rthc_thread_state = sign_registered;
    TRACE("thread registered 0x%" PRIxPTR, osal_thread_self());
    if (rthc_atexit(rthc_thread_dtor, &rthc_thread_state, (void *)&mdbx_version /* dso_anchor */)) {
      ENSURE(pthread_setspecific(rthc_key, &rthc_thread_state) == 0);
      rthc_thread_state = sign_counted;
      const unsigned count_before = atomic_add32(&rthc_pending, 1);
      ENSURE(count_before < INT_MAX);
      NOTICE("fallback to pthreads' tsd, key %" PRIuPTR ", count %u", (uintptr_t)rthc_key, count_before);
      (void)count_before;
    }
  }
  ENSURE(pthread_setspecific(key, value) == 0);
#endif
}

/* dtor called for thread, i.e. for all mdbx's environment objects */
__cold void rthc_thread_dtor(void *rthc) {
  rthc_lock();
  const mdbx_pid_t current_pid = osal_getpid();
#if defined(_WIN32) || defined(_WIN64)
  TRACE(">> pid %zd, thread 0x%" PRIxPTR ", module %p", (size_t)current_pid, osal_thread_self(), rthc);
#else
  TRACE(">> pid %zd, thread 0x%" PRIxPTR ", rthc %p", (size_t)current_pid, osal_thread_self(), rthc);
#endif

  for (size_t i = 0; i < rthc_count; ++i) {
    MDBX_env *const env = rthc_table[i].env;
    if (env->pid != current_pid)
      continue;
    if (!(env->flags & ENV_TXKEY))
      continue;
    reader_slot_t *const slot = thread_rthc_get(env->me_txkey);
    reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0];
    reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers];
    if (slot < begin || slot >= end)
      continue;
#if !defined(_WIN32) && !defined(_WIN64)
    if (pthread_setspecific(env->me_txkey, nullptr) != 0) {
      TRACE("== thread 0x%" PRIxPTR ", rthc %p: ignore race with tsd-key deletion", osal_thread_self(),
            __Wpedantic_format_voidptr(slot));
      continue /* ignore race with tsd-key deletion by mdbx_env_close() */;
    }
#endif

    TRACE("== thread 0x%" PRIxPTR ", rthc %p, [%zi], %p ... %p (%+i), rtch-pid %zi, "
          "current-pid %zi",
          osal_thread_self(), __Wpedantic_format_voidptr(slot), i, __Wpedantic_format_voidptr(begin),
          __Wpedantic_format_voidptr(end), (int)(slot - begin), (size_t)slot->pid.weak, (size_t)current_pid);
    if (atomic_load_pid(&slot->pid, mo_Relaxed) == current_pid) {
      TRACE("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", osal_thread_self(), __Wpedantic_format_voidptr(slot));
      (void)atomic_cas32(&slot->pid, current_pid, 0);
      atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed);
    }
  }

#if defined(_WIN32) || defined(_WIN64)
  TRACE("<< thread 0x%" PRIxPTR ", module %p", osal_thread_self(), rthc);
  rthc_unlock();
#else
  const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc);
  const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc);
  const uint64_t state = rthc_read(rthc);
  if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) {
    TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %zd, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc,
          (size_t)osal_getpid(), "registered", state);
  } else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) {
    TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %zd, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc,
          (size_t)osal_getpid(), "counted", state);
    ENSURE(atomic_sub32(&rthc_pending, 1) > 0);
  } else {
    WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %zd, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc,
            (size_t)osal_getpid(), "wrong", state);
  }

  if (atomic_load32(&rthc_pending, mo_AcquireRelease) == 0) {
    TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %zd, wake", osal_thread_self(), rthc, (size_t)osal_getpid());
    ENSURE(pthread_cond_broadcast(&rthc_cond) == 0);
  }

  TRACE("<< thread 0x%" PRIxPTR ", rthc %p", osal_thread_self(), rthc);
  /* Allow tail call optimization, i.e. gcc should generate the jmp instruction
   * instead of a call for pthread_mutex_unlock() and therefore CPU could not
   * return to current DSO's code section, which may be unloaded immediately
   * after the mutex got released. */
  pthread_mutex_unlock(&rthc_mutex);
#endif
}

__cold int rthc_register(MDBX_env *const env) {
  TRACE(">> env %p, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), rthc_count, rthc_limit);

  int rc = MDBX_SUCCESS;
  for (size_t i = 0; i < rthc_count; ++i)
    if (unlikely(rthc_table[i].env == env)) {
      rc = MDBX_PANIC;
      goto bailout;
    }

  env->me_txkey = 0;
  if (unlikely(rthc_count == rthc_limit)) {
    rthc_entry_t *new_table =
        osal_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table, sizeof(rthc_entry_t) * rthc_limit * 2);
    if (unlikely(new_table == nullptr)) {
      rc = MDBX_ENOMEM;
      goto bailout;
    }
    if (rthc_table == rthc_table_static)
      memcpy(new_table, rthc_table, sizeof(rthc_entry_t) * rthc_limit);
    rthc_table = new_table;
    rthc_limit *= 2;
  }

  if ((env->flags & MDBX_NOSTICKYTHREADS) == 0) {
    rc = thread_key_create(&env->me_txkey);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;
    env->flags |= ENV_TXKEY;
  }

  rthc_table[rthc_count].env = env;
  TRACE("== [%i] = env %p, key %" PRIuPTR, rthc_count, __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey);
  ++rthc_count;

bailout:
  TRACE("<< env %p, key %" PRIuPTR ", rthc_count %u, rthc_limit %u, rc %d", __Wpedantic_format_voidptr(env),
        (uintptr_t)env->me_txkey, rthc_count, rthc_limit, rc);
  return rc;
}

__cold static int rthc_drown(MDBX_env *const env) {
  const mdbx_pid_t current_pid = osal_getpid();
  int rc = MDBX_SUCCESS;
  MDBX_env *inprocess_neighbor = nullptr;
  if (likely(env->lck_mmap.lck && current_pid == env->pid)) {
    rc = rthc_uniq_check(&env->lck_mmap, &inprocess_neighbor);
    if (!inprocess_neighbor) {
      reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0];
      reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers];
      TRACE("== %s env %p pid %zd, readers %p ...%p, current-pid %zd", (current_pid == env->pid) ? "cleanup" : "skip",
            __Wpedantic_format_voidptr(env), (size_t)env->pid, __Wpedantic_format_voidptr(begin),
            __Wpedantic_format_voidptr(end), (size_t)current_pid);
      bool cleaned = false;
      for (reader_slot_t *slot = begin; slot < end; ++slot) {
        if (atomic_load_pid(&slot->pid, mo_Relaxed) == current_pid) {
          (void)atomic_cas32(&slot->pid, current_pid, 0);
          TRACE("== cleanup %p", __Wpedantic_format_voidptr(slot));
          cleaned = true;
        }
      }
      if (cleaned)
        atomic_store32(&env->lck_mmap.lck->rdt_refresh_flag, true, mo_Relaxed);
      if (env->registered_reader_pid && env->lck_mmap.fd != INVALID_HANDLE_VALUE) {
        int err = lck_rpid_clear(env);
        rc = rc ? rc : err;
      }
    }
  }
  int err = lck_destroy(env, inprocess_neighbor, current_pid);
  env->pid = 0;
  return rc ? rc : err;
}

__cold int rthc_remove(MDBX_env *const env) {
  TRACE(">>> env %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey,
        rthc_count, rthc_limit);

  int rc = MDBX_SUCCESS;
  if (likely(env->pid))
    rc = rthc_drown(env);

  for (size_t i = 0; i < rthc_count; ++i) {
    if (rthc_table[i].env == env) {
      if (--rthc_count > 0)
        rthc_table[i] = rthc_table[rthc_count];
      else if (rthc_table != rthc_table_static) {
        void *tmp = rthc_table;
        rthc_table = rthc_table_static;
        rthc_limit = RTHC_INITIAL_LIMIT;
        osal_memory_barrier();
        osal_free(tmp);
      }
      break;
    }
  }

  TRACE("<<< %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey,
        rthc_count, rthc_limit);
  return rc;
}

#if !defined(_WIN32) && !defined(_WIN64)
__cold void rthc_afterfork(void) {
  NOTICE("drown %d rthc entries", rthc_count);
  for (size_t i = 0; i < rthc_count; ++i) {
    MDBX_env *const env = rthc_table[i].env;
    NOTICE("drown env %p", __Wpedantic_format_voidptr(env));
    if (env->lck_mmap.lck)
      osal_munmap(&env->lck_mmap);
    if (env->dxb_mmap.base) {
      osal_munmap(&env->dxb_mmap);
#ifdef ENABLE_MEMCHECK
      VALGRIND_DISCARD(env->valgrind_handle);
      env->valgrind_handle = -1;
#endif /* ENABLE_MEMCHECK */
    }
    env->lck = lckless_stub(env);
    rthc_drown(env);
  }
  if (rthc_table != rthc_table_static)
    osal_free(rthc_table);
  rthc_count = 0;
  rthc_table = rthc_table_static;
  rthc_limit = RTHC_INITIAL_LIMIT;
  rthc_pending.weak = 0;
}
#endif /* ! Windows */

__cold void rthc_ctor(void) {
#if defined(_WIN32) || defined(_WIN64)
  InitializeCriticalSection(&rthc_critical_section);
#else
  ENSURE(pthread_atfork(nullptr, nullptr, rthc_afterfork) == 0);
  ENSURE(pthread_key_create(&rthc_key, rthc_thread_dtor) == 0);
  TRACE("pid %d, &mdbx_rthc_key = %p, value 0x%x", osal_getpid(), __Wpedantic_format_voidptr(&rthc_key),
        (unsigned)rthc_key);
#endif
}

__cold void rthc_dtor(const mdbx_pid_t current_pid) {
  rthc_lock();
#if !defined(_WIN32) && !defined(_WIN64)
  uint64_t *rthc = pthread_getspecific(rthc_key);
  TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %zd, self-status 0x%08" PRIx64 ", left %d", osal_thread_self(),
        __Wpedantic_format_voidptr(rthc), (size_t)current_pid, rthc ? rthc_read(rthc) : ~UINT64_C(0),
        atomic_load32(&rthc_pending, mo_Relaxed));
  if (rthc) {
    const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc);
    const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc);
    const uint64_t state = rthc_read(rthc);
    if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) {
      TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %zd, self-status %s (0x%08" PRIx64 ")", osal_thread_self(),
            __Wpedantic_format_voidptr(rthc), (size_t)current_pid, "registered", state);
    } else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) {
      TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %zd, self-status %s (0x%08" PRIx64 ")", osal_thread_self(),
            __Wpedantic_format_voidptr(rthc), (size_t)current_pid, "counted", state);
      ENSURE(atomic_sub32(&rthc_pending, 1) > 0);
    } else {
      WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %zd, self-status %s (0x%08" PRIx64 ")", osal_thread_self(),
              __Wpedantic_format_voidptr(rthc), (size_t)current_pid, "wrong", state);
    }
  }

  struct timespec abstime;
  ENSURE(clock_gettime(CLOCK_REALTIME, &abstime) == 0);
  abstime.tv_nsec += 1000000000l / 10;
  if (abstime.tv_nsec >= 1000000000l) {
    abstime.tv_nsec -= 1000000000l;
    abstime.tv_sec += 1;
  }
#if MDBX_DEBUG > 0
  abstime.tv_sec += 600;
#endif

  for (unsigned left; (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) {
    NOTICE("cleanup thread-local-dtors: pid %zd, pending %u, wait for...", (size_t)current_pid, left);
    const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime);
    if (rc && rc != EINTR)
      break;
  }
  thread_key_delete(rthc_key);
#endif

  for (size_t i = 0; i < rthc_count; ++i) {
    MDBX_env *const env = rthc_table[i].env;
    if (env->pid != current_pid)
      continue;
    if (!env->lck_mmap.lck || env->lck_mmap.base == MAP_FAILED)
      continue;
    if (!(env->flags & ENV_TXKEY) || !env->lck_mmap.lck)
      continue;
    env->flags -= ENV_TXKEY;
    reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0];
    reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers];
    thread_key_delete(env->me_txkey);
    bool cleaned = false;
    for (reader_slot_t *slot = begin; slot < end; ++slot) {
      TRACE("== [%zi] = key %" PRIuPTR ", %p ... %p, rthc %p (%+i), "
            "rthc-pid %zi, current-pid %zi",
            i, (uintptr_t)env->me_txkey, __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end),
            __Wpedantic_format_voidptr(slot), (int)(slot - begin), (size_t)slot->pid.weak, (size_t)current_pid);
      if (atomic_load_pid(&slot->pid, mo_Relaxed) == current_pid) {
        (void)atomic_cas32(&slot->pid, current_pid, 0);
        TRACE("== cleanup %p", __Wpedantic_format_voidptr(slot));
        cleaned = true;
      }
    }
    if (cleaned)
      atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed);
  }

  rthc_limit = rthc_count = 0;
  if (rthc_table != rthc_table_static)
    osal_free(rthc_table);
  rthc_table = nullptr;
  rthc_unlock();

#if defined(_WIN32) || defined(_WIN64)
  DeleteCriticalSection(&rthc_critical_section);
#else
  /* LY: yielding a few timeslices to give a more chance
   * to racing destructor(s) for completion. */
  workaround_glibc_bug21031();
#endif
}

void spill_remove(MDBX_txn *txn, size_t idx, size_t npages) {
  cASSERT0(txn, idx > 0 && idx <= pnl_size(txn->wr.spilled.list) && txn->wr.spilled.least_removed > 0);
  txn->wr.spilled.least_removed = (idx < txn->wr.spilled.least_removed) ? idx : txn->wr.spilled.least_removed;
  txn->wr.spilled.list[idx] |= 1;
  pnl_setsize(txn->wr.spilled.list, pnl_size(txn->wr.spilled.list) - (idx == pnl_size(txn->wr.spilled.list)));

  while (unlikely(npages > 1)) {
    const pgno_t pgno = (txn->wr.spilled.list[idx] >> 1) + 1;
    if (MDBX_PNL_ASCENDING) {
      if (++idx > pnl_size(txn->wr.spilled.list) || (txn->wr.spilled.list[idx] >> 1) != pgno)
        return;
    } else {
      if (--idx < 1 || (txn->wr.spilled.list[idx] >> 1) != pgno)
        return;
      txn->wr.spilled.least_removed = (idx < txn->wr.spilled.least_removed) ? idx : txn->wr.spilled.least_removed;
    }
    txn->wr.spilled.list[idx] |= 1;
    pnl_setsize(txn->wr.spilled.list, pnl_size(txn->wr.spilled.list) - (idx == pnl_size(txn->wr.spilled.list)));
    --npages;
  }
}

pnl_t spill_purge(MDBX_txn *txn) {
  cASSERT0(txn, txn->wr.spilled.least_removed > 0);
  const pnl_t sl = txn->wr.spilled.list;
  if (txn->wr.spilled.least_removed != INT_MAX) {
    size_t len = pnl_size(sl), r, w;
    for (w = r = txn->wr.spilled.least_removed; r <= len; ++r) {
      sl[w] = sl[r];
      w += 1 - (sl[r] & 1);
    }
    for (size_t i = 1; i < w; ++i)
      cASSERT0(txn, (sl[i] & 1) == 0);
    pnl_setsize(sl, w - 1);
    txn->wr.spilled.least_removed = INT_MAX;
  } else {
    for (size_t i = 1; i <= pnl_size(sl); ++i)
      cASSERT0(txn, (sl[i] & 1) == 0);
  }
  return sl;
}

/*----------------------------------------------------------------------------*/

static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, const size_t npages) {
  cASSERT0(txn, !(txn->flags & MDBX_WRITEMAP));
  if (MDBX_ENABLE_PGOP_STAT)
    txn->env->lck->pgops.spill.weak += npages;
  const pgno_t pgno = dp->pgno;
  int err = iov_page(txn, ctx, dp, npages);
  if (likely(err == MDBX_SUCCESS))
    err = spill_append_span(&txn->wr.spilled.list, pgno, npages);
  return err;
}

/* Set unspillable LRU-label for dirty pages watched by txn.
 * Returns the number of pages marked as unspillable. */
static size_t spill_cursor_keep(const MDBX_txn *const txn, const MDBX_cursor *mc) {
  cASSERT0(txn, (txn->flags & (txn_ro_both | MDBX_WRITEMAP)) == 0);
  size_t keep = 0;
  while (!is_poor(mc)) {
    cASSERT0(txn, mc->top >= 0);
    const page_t *mp;
    intptr_t i = 0;
    do {
      mp = mc->pg[i];
      TRACE("dbi %zu, mc-%p[%zu], page %u %p", cursor_dbi(mc), __Wpedantic_format_voidptr(mc), i, mp->pgno,
            __Wpedantic_format_voidptr(mp));
      cASSERT0(txn, !is_subpage(mp));
      if (is_modifable(txn, mp)) {
        size_t const n = txn_dpl_search(txn, mp->pgno);
        if (txn->wr.dirtylist->items[n].pgno == mp->pgno &&
            /* не считаем дважды */ txn_dpl_age(txn, n)) {
          size_t *const ptr = ptr_disp(txn->wr.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
          *ptr = txn->wr.dirtylru;
          cASSERT0(txn, txn_dpl_age(txn, n) == 0);
          ++keep;
          DEBUG("keep page %" PRIaPGNO " (%p), dbi %zu, %scursor %p[%zu]", mp->pgno, __Wpedantic_format_voidptr(mp),
                cursor_dbi(mc), is_inner(mc) ? "sub-" : "", __Wpedantic_format_voidptr(mc), i);
        }
      }
    } while (++i <= mc->top);

    cASSERT0(txn, is_leaf(mp));
    if (!inner_pointed(mc))
      break;
    mc = &mc->subcur->cursor;
    if (is_subpage(mc->pg[0]))
      break;
  }
  return keep;
}

static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) {
  cASSERT0(txn, (txn->flags & (txn_ro_both | MDBX_WRITEMAP)) == 0);
  txn_dpl_lru_turn(txn);
  size_t keep = m0 ? spill_cursor_keep(txn, m0) : 0;

  TXN_FOREACH_DBI_ALL(txn, dbi) {
    if (F_ISSET(txn->dbi_state[dbi], DBI_DIRTY | DBI_VALID) && txn->dbs[dbi].root != P_INVALID)
      for (MDBX_cursor *mc = txn->cursors[dbi]; mc; mc = mc->next)
        if (mc != m0)
          keep += spill_cursor_keep(txn, mc);
  }

  return keep;
}

/* Returns the spilling priority (0..255) for a dirty page:
 *      0 = should be spilled;
 *    ...
 *  > 255 = must not be spilled. */
MDBX_NOTHROW_PURE_FUNCTION static unsigned spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) {
  dpl_t *const dl = txn->wr.dirtylist;
  const uint32_t age = txn_dpl_age(txn, i);
  const size_t npages = dpl_npages(dl, i);
  const pgno_t pgno = dl->items[i].pgno;
  if (age == 0) {
    DEBUG("skip %s %zu page %" PRIaPGNO, "keep", npages, pgno);
    return 256;
  }

  page_t *const dp = dl->items[i].ptr;
  if (dp->flags & (P_LOOSE | P_SPILLED)) {
    DEBUG("skip %s %zu page %" PRIaPGNO, (dp->flags & P_LOOSE) ? "loose" : "parent-spilled", npages, pgno);
    return 256;
  }

  /* Can't spill twice,
   * make sure it's not already in a parent's spill list(s). */
  MDBX_txn *parent = txn->parent;
  if (parent && (parent->flags & MDBX_TXN_SPILLS)) {
    do
      if (spill_intersect(parent, pgno, npages)) {
        DEBUG("skip-2 parent-spilled %zu page %" PRIaPGNO, npages, pgno);
        dp->flags |= P_SPILLED;
        return 256;
      }
    while ((parent = parent->parent) != nullptr);
  }

  cASSERT0(txn, age * (uint64_t)reciprocal < UINT32_MAX);
  unsigned prio = age * reciprocal >> 24;
  cASSERT0(txn, prio < 256);
  if (likely(npages == 1))
    return prio = 256 - prio;

  /* make a large/overflow pages be likely to spill */
  size_t factor = npages | npages >> 1;
  factor |= factor >> 2;
  factor |= factor >> 4;
  factor |= factor >> 8;
  factor |= factor >> 16;
  factor = (size_t)prio * log2n_powerof2(factor + 1) + /* golden ratio */ 157;
  factor = (factor < 256) ? 255 - factor : 0;
  cASSERT0(txn, factor < 256 && factor < (256 - prio));
  return prio = (unsigned)factor;
}

static size_t spill_gate(const MDBX_env *env, intptr_t part, const size_t total) {
  const intptr_t spill_min = env->options.spill_min_denominator
                                 ? (total + env->options.spill_min_denominator - 1) / env->options.spill_min_denominator
                                 : 1;
  const intptr_t spill_max =
      total - (env->options.spill_max_denominator ? total / env->options.spill_max_denominator : 0);
  part = (part < spill_max) ? part : spill_max;
  part = (part > spill_min) ? part : spill_min;
  eASSERT0(env, part >= 0 && (size_t)part <= total);
  return (size_t)part;
}

__cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries,
                          const intptr_t wanna_spill_npages, const size_t need) {
  cASSERT0(txn, (txn->flags & txn_ro_both) == 0);

  int rc = MDBX_SUCCESS;
  if (unlikely(txn->wr.loose_count >=
               (txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages)))
    goto done;

  const size_t dirty_entries = txn->wr.dirtylist ? (txn->wr.dirtylist->length - txn->wr.loose_count) : 1;
  const size_t dirty_npages =
      (txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages) -
      txn->wr.loose_count;
  const size_t need_spill_entries = spill_gate(txn->env, wanna_spill_entries, dirty_entries);
  const size_t need_spill_npages = spill_gate(txn->env, wanna_spill_npages, dirty_npages);

  const size_t need_spill = (need_spill_entries > need_spill_npages) ? need_spill_entries : need_spill_npages;
  if (!need_spill)
    goto done;

  if (txn->flags & MDBX_WRITEMAP) {
    NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", dirty_entries, dirty_npages);
    const MDBX_env *env = txn->env;
    cASSERT0(txn, txn->wr.spilled.list == nullptr);
    rc = dxb_msync(txn->env, txn->geo.first_unallocated, MDBX_SYNC_KICK);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;
#if MDBX_AVOID_MSYNC
    MDBX_ANALYSIS_ASSUME(txn->wr.dirtylist != nullptr);
    tASSERT1(txn, txn_dpl_check(txn));
    env->lck->unsynced_pages.weak += txn->wr.dirtylist->pages_including_loose - txn->wr.loose_count;
    dpl_setlen(txn->wr.dirtylist, 0);
    txn->wr.dirtyroom = env->options.dp_limit - txn->wr.loose_count;
    for (page_t *lp = txn->wr.loose_pages; lp != nullptr; lp = page_next(lp)) {
      cASSERT0(txn, lp->flags == P_LOOSE);
      rc = txn_dpl_append(txn, lp->pgno, lp, 1);
      if (unlikely(rc != MDBX_SUCCESS))
        goto bailout;
      MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
      VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
    }
    tASSERT1(txn, txn_dpl_check(txn));
#else
    cASSERT0(txn, txn->wr.dirtylist == nullptr);
    env->lck->unsynced_pages.weak += txn->wr.writemap_dirty_npages;
    txn->wr.writemap_spilled_npages += txn->wr.writemap_dirty_npages;
    txn->wr.writemap_dirty_npages = 0;
#endif /* MDBX_AVOID_MSYNC */
    goto done;
  }

  NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", need_spill_entries, need_spill_npages);
  MDBX_ANALYSIS_ASSUME(txn->wr.dirtylist != nullptr);
  cASSERT0(txn, txn->wr.dirtylist->length - txn->wr.loose_count >= 1);
  cASSERT0(txn, txn->wr.dirtylist->pages_including_loose - txn->wr.loose_count >= need_spill_npages);
  if (!txn->wr.spilled.list) {
    txn->wr.spilled.least_removed = INT_MAX;
    txn->wr.spilled.list = pnl_alloc(need_spill);
    if (unlikely(!txn->wr.spilled.list)) {
      rc = MDBX_ENOMEM;
    bailout:
      txn->flags |= MDBX_TXN_ERROR;
      return rc;
    }
  } else {
    /* purge deleted slots */
    spill_purge(txn);
    rc = pnl_reserve(&txn->wr.spilled.list, need_spill);
    (void)rc /* ignore since the resulting list may be shorter
     and pnl_append() will increase pnl on demand */
        ;
  }

  /* Сортируем чтобы запись на диск была полее последовательна */
  dpl_t *const dl = txn_dpl_sort(txn);

  /* Preserve pages which may soon be dirtied again */
  const size_t unspillable = spill_txn_keep(txn, m0);
  if (unspillable + txn->wr.loose_count >= dl->length) {
#if xMDBX_DEBUG_SPILLING == 1 /* avoid false failure in debug mode  */
    if (likely(txn->wr.dirtyroom + txn->wr.loose_count >= need))
      return MDBX_SUCCESS;
#endif /* xMDBX_DEBUG_SPILLING */
    ERROR("all %zu dirty pages are unspillable since referenced "
          "by a cursor(s), use fewer cursors or increase "
          "MDBX_opt_txn_dp_limit",
          unspillable);
    goto done;
  }

  /* Подзадача: Вытолкнуть часть страниц на диск в соответствии с LRU,
   * но при этом учесть важные поправки:
   *  - лучше выталкивать старые large/overflow страницы, так будет освобождено
   *    больше памяти, а также так как они (в текущем понимании) гораздо реже
   *    повторно изменяются;
   *  - при прочих равных лучше выталкивать смежные страницы, так будет
   *    меньше I/O операций;
   *  - желательно потратить на это меньше времени чем std::partial_sort_copy;
   *
   * Решение:
   *  - Квантуем весь диапазон lru-меток до 256 значений и задействуем один
   *    проход 8-битного radix-sort. В результате получаем 256 уровней
   *    "свежести", в том числе значение lru-метки, старее которой страницы
   *    должны быть выгружены;
   *  - Двигаемся последовательно в сторону увеличения номеров страниц
   *    и выталкиваем страницы с lru-меткой старее отсекающего значения,
   *    пока не вытолкнем достаточно;
   *  - Встречая страницы смежные с выталкиваемыми для уменьшения кол-ва
   *    I/O операций выталкиваем и их, если они попадают в первую половину
   *    между выталкиваемыми и самыми свежими lru-метками;
   *  - дополнительно при сортировке умышленно старим large/overflow страницы,
   *    тем самым повышая их шансы на выталкивание. */

  /* get min/max of LRU-labels */
  uint32_t age_max = 0;
  for (size_t i = 1; i <= dl->length; ++i) {
    const uint32_t age = txn_dpl_age(txn, i);
    age_max = (age_max >= age) ? age_max : age;
  }

  VERBOSE("lru-head %u, age-max %u", txn->wr.dirtylru, age_max);

  /* half of 8-bit radix-sort */
  pgno_t radix_entries[256], radix_npages[256];
  memset(&radix_entries, 0, sizeof(radix_entries));
  memset(&radix_npages, 0, sizeof(radix_npages));
  size_t spillable_entries = 0, spillable_npages = 0;
  const uint32_t reciprocal = (UINT32_C(255) << 24) / (age_max + 1);
  for (size_t i = 1; i <= dl->length; ++i) {
    const unsigned prio = spill_prio(txn, i, reciprocal);
    size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
    TRACE("page %" PRIaPGNO ", lru %zu, is_multi %c, npages %u, age %u of %u, prio %u", dl->items[i].pgno, *ptr,
          (dpl_npages(dl, i) > 1) ? 'Y' : 'N', dpl_npages(dl, i), txn_dpl_age(txn, i), age_max, prio);
    if (prio < 256) {
      radix_entries[prio] += 1;
      spillable_entries += 1;
      const pgno_t npages = dpl_npages(dl, i);
      radix_npages[prio] += npages;
      spillable_npages += npages;
    }
  }

  cASSERT0(txn, spillable_npages >= spillable_entries);
  pgno_t spilled_entries = 0, spilled_npages = 0;
  if (likely(spillable_entries > 0)) {
    size_t prio2spill = 0, prio2adjacent = 128, amount_entries = radix_entries[0], amount_npages = radix_npages[0];
    for (size_t i = 1; i < 256; i++) {
      if (amount_entries < need_spill_entries || amount_npages < need_spill_npages) {
        prio2spill = i;
        prio2adjacent = i + (257 - i) / 2;
        amount_entries += radix_entries[i];
        amount_npages += radix_npages[i];
      } else if (amount_entries + amount_entries < spillable_entries + need_spill_entries
                 /* РАВНОЗНАЧНО: amount - need_spill < spillable - amount */
                 || amount_npages + amount_npages < spillable_npages + need_spill_npages) {
        prio2adjacent = i;
        amount_entries += radix_entries[i];
        amount_npages += radix_npages[i];
      } else
        break;
    }

    VERBOSE("prio2spill %zu, prio2adjacent %zu, spillable %zu/%zu,"
            " wanna-spill %zu/%zu, amount %zu/%zu",
            prio2spill, prio2adjacent, spillable_entries, spillable_npages, need_spill_entries, need_spill_npages,
            amount_entries, amount_npages);
    cASSERT0(txn, prio2spill < prio2adjacent && prio2adjacent <= 256);

    iov_ctx_t ctx;
    rc = iov_init(txn, &ctx, amount_entries, amount_npages,
#if defined(_WIN32) || defined(_WIN64)
                  txn->env->ioring.overlapped_fd ? txn->env->ioring.overlapped_fd :
#endif
                                                 txn->env->lazy_fd,
                  true);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;

    size_t r = 0, w = 0;
    pgno_t last = 0;
    while (r < dl->length && (spilled_entries < need_spill_entries || spilled_npages < need_spill_npages)) {
      dl->items[++w] = dl->items[++r];
      unsigned prio = spill_prio(txn, w, reciprocal);
      if (prio > prio2spill && (prio >= prio2adjacent || last != dl->items[w].pgno))
        continue;

      const size_t e = w;
      last = dpl_endpgno(dl, w);
      while (--w && dpl_endpgno(dl, w) == dl->items[w + 1].pgno && spill_prio(txn, w, reciprocal) < prio2adjacent)
        ;

      for (size_t i = w; ++i <= e;) {
        const unsigned npages = dpl_npages(dl, i);
        prio = spill_prio(txn, i, reciprocal);
        DEBUG("%sspill[%zu] %u page %" PRIaPGNO " (age %d, prio %u)", (prio > prio2spill) ? "co-" : "", i, npages,
              dl->items[i].pgno, txn_dpl_age(txn, i), prio);
        cASSERT0(txn, prio < 256);
        ++spilled_entries;
        spilled_npages += npages;
        rc = spill_page(txn, &ctx, dl->items[i].ptr, npages);
        if (unlikely(rc != MDBX_SUCCESS))
          goto failed;
      }
    }

    VERBOSE("spilled entries %u, spilled npages %u", spilled_entries, spilled_npages);
    cASSERT0(txn, spillable_entries == 0 || spilled_entries > 0);
    cASSERT0(txn, spilled_npages >= spilled_entries);

  failed:
    while (r < dl->length)
      dl->items[++w] = dl->items[++r];
    cASSERT0(txn, r - w == spilled_entries || rc != MDBX_SUCCESS);

    dl->sorted = dpl_setlen(dl, w);
    txn->wr.dirtyroom += spilled_entries;
    txn->wr.dirtylist->pages_including_loose -= spilled_npages;
    tASSERT1(txn, txn_dpl_check(txn));

    if (!iov_empty(&ctx)) {
      cASSERT0(txn, rc == MDBX_SUCCESS);
      rc = iov_write(&ctx);
    }
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;

    txn->env->lck->unsynced_pages.weak += spilled_npages;
    pnl_sort(txn->wr.spilled.list, (size_t)txn->geo.first_unallocated << 1);
    txn->flags |= MDBX_TXN_SPILLS;
    NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", spilled_entries, spilled_npages,
           txn->wr.dirtyroom);
  } else {
    cASSERT0(txn, rc == MDBX_SUCCESS);
    for (size_t i = 1; i <= dl->length; ++i) {
      page_t *dp = dl->items[i].ptr;
      VERBOSE("unspillable[%zu]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", i, dp->pgno, dpl_npages(dl, i),
              dp->flags, txn_dpl_age(txn, i), spill_prio(txn, i, reciprocal));
    }
  }

#if xMDBX_DEBUG_SPILLING == 2
  if (txn->wr.loose_count + txn->wr.dirtyroom <= need / 2 + 1)
    ERROR("dirty-list length: before %zu, after %zu, parent %zi, loose %zu; "
          "needed %zu, spillable %zu; "
          "spilled %u dirty-entries, now have %zu dirty-room",
          dl->length + spilled_entries, dl->length,
          (txn->parent && txn->parent->wr.dirtylist) ? (intptr_t)txn->parent->wr.dirtylist->length : -1,
          txn->wr.loose_count, need, spillable_entries, spilled_entries, txn->wr.dirtyroom);
  ENSURE(txn->env, txn->wr.loose_count + txn->wr.dirtyroom > need / 2);
#endif /* xMDBX_DEBUG_SPILLING */

done:
  return likely(txn->wr.dirtyroom + txn->wr.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need))
             ? MDBX_SUCCESS
             : MDBX_TXN_FULL;
}

__hot txnid_t tbl_root_txnid(const MDBX_txn *txn, const size_t dbi) {
  if ((txn->flags & txn_ro_flat) == 0) {
    const MDBX_txn *scan = txn;
    do
      if ((scan->flags & MDBX_TXN_DIRTY) && (dbi == MAIN_DBI || (scan->dbi_state[dbi] & DBI_DIRTY))) {
        /* После коммита вложенных тразакций может быть mod_txnid > front */
        return scan->front_txnid;
      }
    while (unlikely((scan = scan->parent) != nullptr));
  }
  return /* tree->mod_txnid maybe zero in a legacy DB */ txn->dbs[dbi].mod_txnid ? txn->dbs[dbi].mod_txnid : txn->txnid;
}

int tbl_setup(const MDBX_env *env, volatile kvx_t *const kvx, const tree_t *const db) {
  osal_memory_fence(mo_AcquireRelease, false);

  if (unlikely(!check_table_flags(db->flags))) {
    ERROR("incompatible or invalid db.flags (0x%x) ", db->flags);
    return MDBX_INCOMPATIBLE;
  }

  size_t v_lmin = valsize_min(db->flags);
  size_t v_lmax = env_valsize_max(env, db->flags);
  if ((db->flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->dupfix_size) {
    if (!MDBX_DISABLE_VALIDATION && unlikely(db->dupfix_size < v_lmin || db->dupfix_size > v_lmax)) {
      ERROR("db.dupfix_size (%u) <> min/max value-length (%zu/%zu)", db->dupfix_size, v_lmin, v_lmax);
      return MDBX_CORRUPTED;
    }
    v_lmin = v_lmax = db->dupfix_size;
  }

  kvx->clc.k.lmin = keysize_min(db->flags);
  kvx->clc.k.lmax = env_keysize_max(env, db->flags);
  clc_reset_methods(&kvx->clc.k);
  clc_reset_methods(&kvx->clc.v);
  if (unlikely(!kvx->clc.k.cmp)) {
    kvx->clc.v.cmp = builtin_datacmp(db->flags);
    kvx->clc.k.cmp = builtin_keycmp(db->flags);
  }
  kvx->clc.v.lmin = v_lmin;
  osal_memory_fence(mo_Relaxed, true);
  kvx->clc.v.lmax = v_lmax;
  osal_memory_fence(mo_AcquireRelease, true);

  eASSERT0(env, kvx->clc.k.lmax >= kvx->clc.k.lmin);
  eASSERT0(env, kvx->clc.v.lmax >= kvx->clc.v.lmin);
  return MDBX_SUCCESS;
}

int tbl_fetch(MDBX_txn *txn, MDBX_cursor *mc, size_t dbi, const MDBX_val *name, unsigned wanna_flags) {
  int err = cursor_init(mc, txn, MAIN_DBI);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  err = tree_search(mc, name, 0);
  if (unlikely(err != MDBX_SUCCESS)) {
    if (err == MDBX_NOTFOUND)
      goto notfound;
    return err;
  }

  sfr_t sr = tree_search_foliage(mc, name);
  if (unlikely(!sr.exact)) {
  notfound:
    if (dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) && !(wanna_flags & MDBX_CREATE))
      NOTICE("dbi %zu refs to non-existing table `%.*s` for txn %" PRIaTXN " (err %d)", dbi, (int)name->iov_len,
             (const char *)name->iov_base, txn->txnid, err);
    return MDBX_NOTFOUND;
  }

  if (unlikely((node_flags(sr.node) & (N_DUP | N_TREE)) != N_TREE)) {
    NOTICE("dbi %zu refs to not a named table `%.*s` for txn %" PRIaTXN " (%s)", dbi, (int)name->iov_len,
           (const char *)name->iov_base, txn->txnid, "wrong node-flags");
    return MDBX_INCOMPATIBLE /* not a named DB */;
  }

  MDBX_val data;
  err = node_read(mc, sr.node, &data, mc->pg[mc->top]);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  if (unlikely(data.iov_len < sizeof(tree_t))) {
    NOTICE("dbi %zu refs to not a named table `%.*s` for txn %" PRIaTXN " (%s)", dbi, (int)name->iov_len,
           (const char *)name->iov_base, txn->txnid, "wrong record-size");
    return MDBX_INCOMPATIBLE /* not a named DB */;
  }

  const unsigned db_flags = UNALIGNED_PEEK_16(data.iov_base, tree_t, flags);
  const pgno_t db_root_pgno = peek_pgno(ptr_disp(data.iov_base, offsetof(tree_t, root)));
  /* The txn may not know this DBI, or another process may
   * have dropped and recreated the DB with other flags. */
  if (unlikely((wanna_flags ^ db_flags) & DB_PERSISTENT_FLAGS) && !(wanna_flags & MDBX_DB_ACCEDE) &&
      !((wanna_flags & MDBX_CREATE) && db_root_pgno == P_INVALID)) {
    NOTICE("dbi %zu refs to the re-created table `%.*s` for txn %" PRIaTXN
           " with different flags (present 0x%X != wanna 0x%X)",
           dbi, (int)name->iov_len, (const char *)name->iov_base, txn->txnid, db_flags & DB_PERSISTENT_FLAGS,
           wanna_flags & DB_PERSISTENT_FLAGS);
    return MDBX_INCOMPATIBLE /* not a named DB */;
  }

  tree_t *const db = &txn->dbs[dbi];
  memcpy(db, data.iov_base, sizeof(tree_t));
#if !MDBX_DISABLE_VALIDATION
  const txnid_t maindb_leafpage_txnid = mc->pg[mc->top]->txnid;
  cASSERT0(txn, txn->front_txnid >= maindb_leafpage_txnid);
  if (unlikely(db->mod_txnid > maindb_leafpage_txnid)) {
    ERROR("db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", db->mod_txnid, maindb_leafpage_txnid);
    return MDBX_CORRUPTED;
  }
#endif /* !MDBX_DISABLE_VALIDATION */

  return MDBX_SUCCESS;
}

int tbl_create(MDBX_txn *txn, MDBX_cursor *mc, size_t slot, const MDBX_val *name, unsigned db_flags) {
  cASSERT0(txn, db_flags & MDBX_CREATE);
  MDBX_val body;
  body.iov_base = memset(&txn->dbs[slot], 0, body.iov_len = sizeof(tree_t));
  txn->dbs[slot].root = P_INVALID;
  txn->dbs[slot].mod_txnid = txn->txnid;
  txn->dbs[slot].flags = db_flags & DB_PERSISTENT_FLAGS;
  mc->next = txn->cursors[MAIN_DBI];
  txn->cursors[MAIN_DBI] = mc;
  int err = cursor_put_checklen(mc, name, &body, N_TREE | MDBX_NOOVERWRITE);
  txn->cursors[MAIN_DBI] = mc->next;
  if (likely(err == MDBX_SUCCESS)) {
    txn->flags |= MDBX_TXN_DIRTY;
    cASSERT0(txn, (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) != 0);
  }
  return err;
}

int tbl_refresh(MDBX_txn *txn, size_t dbi) {
  cursor_couple_t couple;
  kvx_t *const kvx = &txn->env->kvs[dbi];
  int rc = tbl_fetch(txn, &couple.outer, dbi, &kvx->name, txn->dbs[dbi].flags);
  if (likely(rc != MDBX_SUCCESS))
    return dbi_gone(txn, dbi, rc);

  rc = tbl_setup_ifneed(txn->env, kvx, &txn->dbs[dbi]);
  if (unlikely(rc != MDBX_SUCCESS))
    return dbi_gone(txn, dbi, rc);

  if (unlikely(dbi_changed(txn, dbi)))
    return MDBX_BAD_DBI;

  txn->dbi_state[dbi] &= ~DBI_STALE;
  return MDBX_SUCCESS;
}

int tbl_purge(MDBX_cursor *mc) {
  if (mc->tree->height) {
    int err = tree_drop(mc);
    if (unlikely(err != MDBX_SUCCESS)) {
      mc->txn->flags |= MDBX_TXN_ERROR;
      return err;
    }

    /* reset the DB record, mark it dirty */
    *mc->dbi_state |= DBI_DIRTY;
    mc->tree->height = 0;
    mc->tree->branch_pages = 0;
    mc->tree->leaf_pages = 0;
    mc->tree->large_pages = 0;
    mc->tree->items = 0;
    mc->tree->root = P_INVALID;
    mc->tree->sequence = 0;
    /* mc->tree->mod_txnid = txn->txnid; */
    mc->txn->flags |= MDBX_TXN_DIRTY;

    /* Invalidate the dropped DB's cursors */
    const size_t dbi = cursor_dbi(mc);
    for (mc = mc->txn->cursors[dbi]; mc; mc = mc->next)
      be_poor(mc);
  }

  return MDBX_SUCCESS;
}

/*----------------------------------------------------------------------------*/

static void tbl_stat_add(const tree_t *db, MDBX_stat *const st) {
  st->ms_depth += db->height;
  st->ms_branch_pages += db->branch_pages;
  st->ms_leaf_pages += db->leaf_pages;
  st->ms_overflow_pages += db->large_pages;
  st->ms_entries += db->items;
  st->ms_mod_txnid = (st->ms_mod_txnid > db->mod_txnid) ? st->ms_mod_txnid : db->mod_txnid;
}

int tbl_stat_summary(const MDBX_txn *txn, MDBX_stat *st) {
  memset(st, 0, sizeof(*st));

  const MDBX_env *const env = txn->env;
  st->ms_psize = env->ps;
  size_t done = 0;
  TXN_FOREACH_DBI_FROM(txn, dbi,
                       /* assuming GC is internal and not subject for accounting */ MAIN_DBI) {
    if ((txn->dbi_state[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID) {
      done += 1;
      tbl_stat_add(txn->dbs + dbi, st);
    }
  }

  if ((txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT) == 0 &&
      done < txn->dbs[MAIN_DBI].items /* TODO: use `md_subs` field */ + /* MainDB */ 1) {
    cursor_couple_t cx;
    int err = cursor_init(&cx.outer, (MDBX_txn *)txn, MAIN_DBI);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    /* scan and account not opened named tables */
    err = tree_search(&cx.outer, nullptr, Z_FIRST);
    while (err == MDBX_SUCCESS) {
      const page_t *mp = cx.outer.pg[cx.outer.top];
      for (size_t i = 0; i < page_numkeys(mp); i++) {
        const node_t *const node = page_node(mp, i);
        if (node_flags(node) != N_TREE)
          continue;
        if (unlikely(node_ds(node) != sizeof(tree_t))) {
          ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", node_ds(node));
          return MDBX_CORRUPTED;
        }

        /* skip opened and already accounted */
        const MDBX_val name = {node_key(node), node_ks(node)};
        TXN_FOREACH_DBI_USER(txn, dbi)
        if ((txn->dbi_state[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID &&
            env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[dbi].name) == 0)
          goto next;

        tree_t db;
        memcpy(&db, node_data(node), sizeof(db));
        tbl_stat_add(&db, st);
      }
    next:
      err = cursor_sibling_right(&cx.outer);
    }
    if (unlikely(err != MDBX_NOTFOUND))
      return err;
  }

  return MDBX_SUCCESS;
}

#if !defined(NDEBUG)
MDBX_MAYBE_UNUSED static char *cursor_dump_stack(const MDBX_cursor *mc, const char *caption, bool print_subcursor,
                                                 bool print_pageptr) {
  static char buf[1024];
  char *const end = buf + sizeof(buf) - 1;
  char *tail = buf;

loop:
  tail += snprintf(tail, end - tail, "%s[%i]", caption, mc->top);
  for (intptr_t i = 0; i <= mc->top; ++i)
    if (print_pageptr)
      tail +=
          snprintf(tail, end - tail, ".%p#%u->%u", __Wpedantic_format_voidptr(mc->pg[i]), mc->pg[i]->pgno, mc->ki[i]);
    else
      tail += snprintf(tail, end - tail, ".%u", mc->ki[i]);
  if (mc->flags & z_inner)
    tail += snprintf(tail, end - tail, "%s", "-inner");
  if (mc->flags & z_after_delete)
    tail += snprintf(tail, end - tail, "%s", "-after.delete");
  if (mc->flags & z_hollow)
    tail += snprintf(tail, end - tail, "%s", "-hollow");
  else if (mc->flags & z_eof_hard)
    tail += snprintf(tail, end - tail, "%s", "-eof.hard");
  else if (mc->flags & z_eof_soft)
    tail += snprintf(tail, end - tail, "%s", "-eof.soft");

  if (print_subcursor && mc->subcur) {
    caption = "->subcur";
    mc = &mc->subcur->cursor;
    goto loop;
  }
  return buf;
}
#endif /* NDEBUG */

/* mostly for debug */
static int cutoff_leaf(MDBX_cursor *axe, unsigned alldups) {
#ifndef NDEBUG
  DKBUF_DEBUG;
  page_t *mp;
  MDBX_val k = {.iov_base = nullptr, .iov_len = 0}, v = {.iov_base = nullptr, .iov_len = 0};
  MDBX_cursor *blade = nullptr;
  const char *s = nullptr;
  const char *sk = nullptr;
  const char *sv = nullptr;

  if (alldups && inner_pointed(axe)) {
    blade = axe;
    s = "outer.dups-node";
    mp = axe->pg[axe->top];
    node_t *node = page_node(mp, axe->ki[axe->top]);
    k.iov_base = node_key(node);
    k.iov_len = node_ks(node);
    sv = "*";
  } else {
    blade = inner_pointed(axe) ? &axe->subcur->cursor : axe;
    s = inner_pointed(axe) ? "outer.inner.entry" : (is_inner(axe) ? "inner.entry" : "outer.entry");

    if (is_inner(blade)) {
      MDBX_cursor *outer = cursor_outer(blade);
      node_t *node = page_node(outer->pg[outer->top], outer->ki[outer->top]);
      k.iov_base = node_key(node);
      k.iov_len = node_ks(node);
    }

    mp = blade->pg[blade->top];
    if (is_dupfix_leaf(mp)) {
      v.iov_base = page_dupfix_ptr(mp, blade->ki[blade->top], mp->dupfix_ksize);
      v.iov_len = mp->dupfix_ksize;
      sv = DVAL_DEBUG(&v);
    } else {
      node_t *node = page_node(mp, blade->ki[blade->top]);
      v.iov_len = node_ks(node);
      v.iov_base = node_key(node);
      if (!is_inner(blade)) {
        k = v;
        v.iov_len = node_ds(node);
        v.iov_base = node_data(node);
      }
    }
    sv = DVAL_DEBUG(&v);
  }
  sk = DKEY_DEBUG(&k);

  (void)k;
  (void)v;
  (void)sk;
  (void)sv;
  VERBOSE("cut %s => \"%s\".\"%s\"", cursor_dump_stack(axe, s, true, false), sk, sv);
#endif /* !NDEBUG */
  return cursor_del(axe, alldups);
}

#if MDBX_ENABLE_BUNCHES_REMOVAL

intptr_t tree_diff_level(const MDBX_cursor *left, const MDBX_cursor *right) {
  ASSERT(left->top == right->top);
  for (intptr_t i = 0; i <= left->top; ++i) {
    ASSERT(left->pg[i] == right->pg[i]);
    if (left->ki[i] != right->ki[i])
      return (left->ki[i] < right->ki[i]) ? i : INT_MIN;
  }
  return MDBX_RESULT_TRUE;
}

int tree_cutoff_twig(MDBX_cursor *mc, const pgno_t pgno, size_t deep, txnid_t parent_txnid, const bool whole_tree) {
  /* Пытаемся избежать чтения листовых страниц и связанных с этим page faults. */
  if (whole_tree && (mc->checking & z_pagecheck) == 0 && deep == mc->tree->height && !mc->tree->large_pages &&
      parent_txnid < mc->txn->txnid /* is_frozen (mc->txn, pagent_page) */
      && !mc->subcur && cursor_dbi(mc) != MAIN_DBI) {
    /* Если не запрошена проверка структуры страниц, и узел ссылается по последний уровень дерева,
     * и нет large/overlow страниц, и не может быть вложенных dupsort-деревьев,
     * то можно избежать обращения к листовой странице, при условии что родительская в статусе FROZEN. */
    return page_retire_ex(mc, pgno, nullptr, P_FROZEN + (mc->checking & (P_LEAF | P_DUPFIX)));
  }

  int err;
  const pgr_t pgr = page_get_three(mc, pgno, parent_txnid);
  if (unlikely(pgr.err != MDBX_SUCCESS))
    return pgr.err;

  if (is_branch(pgr.page)) {
    const size_t branch_nkeys = page_numkeys(pgr.page);
    for (size_t i = 0; i < branch_nkeys; ++i) {
      const node_t *const branch_node = page_node(pgr.page, i);
      cASSERT0(mc, node_flags(branch_node) == 0);
      err = tree_cutoff_twig(mc, node_pgno(branch_node), deep + 1, pgr.page->txnid, whole_tree);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }
  } else {
    tASSERT0(mc, is_leaf(pgr.page));
    if (!whole_tree) {
#ifndef NDEBUG
      MDBX_val key;
      if (is_dupfix_leaf(pgr.page)) {
        key.iov_base = page_dupfix_ptr(pgr.page, 0, pgr.page->dupfix_ksize);
        key.iov_len = pgr.page->dupfix_ksize;
      } else {
        node_t *key_node = page_node(pgr.page, 0);
        key.iov_base = node_key(key_node);
        key.iov_len = node_ks(key_node);
      }
      (void)key;
      DKBUF_DEBUG;
      VERBOSE("cut leaf page %u with %zu items (%" PRIi64 "->%" PRIi64 ") \"%s\"", pgr.page->pgno,
              page_numkeys(pgr.page), mc->tree->items, mc->tree->items - page_numkeys(pgr.page), DKEY_DEBUG(&key));
#endif /* NDEBUG */
      mc->tree->items -= page_numkeys(pgr.page);
    }
    if (mc->tree->large_pages || mc->subcur) {
      tASSERT0(mc, !is_dupfix_leaf(pgr.page));
      const size_t leaf_nkeys = page_numkeys(pgr.page);
      for (size_t i = 0; i < leaf_nkeys; ++i) {
        const node_t *const leaf_node = page_node(pgr.page, i);
        if (leaf_node->flags & (N_DUP | N_TREE)) {
          if (unlikely((leaf_node->flags & N_DUP) == 0))
            return /* disallowing implicit table deletion */ MDBX_INCOMPATIBLE;
          err = cursor_push(mc, pgr.page, 0);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
          err = cursor_dupsort_setup(mc, leaf_node, pgr.page);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
          if (!whole_tree) {
#ifndef NDEBUG
            DKBUF_DEBUG;
            MDBX_val key;
            key.iov_base = node_key(leaf_node);
            key.iov_len = node_ks(leaf_node);
            (void)key;
            VERBOSE("cut leaf %s %u with %" PRIi64 " items (%" PRIi64 "->%" PRIi64 ") \"%s\"",
                    (leaf_node->flags & N_TREE) ? "tree" : "sub-page", mc->subcur->cursor.tree->root,
                    mc->subcur->cursor.tree->items, mc->tree->items,
                    mc->tree->items - (mc->subcur->cursor.tree->items - 1), DKEY_DEBUG(&key));
#endif /* NDEBUG */
            mc->tree->items -= mc->subcur->cursor.tree->items - 1;
          }
          if (leaf_node->flags & N_TREE) {
            MDBX_cursor *const inner = &mc->subcur->cursor;
            err = tree_cutoff_twig(inner, inner->tree->root, 1, tbl_root_txnid(mc->txn, cursor_dbi(mc)), true);
            if (unlikely(err != MDBX_SUCCESS))
              return err;
          }
          cursor_pop(mc);
        } else if (leaf_node->flags & N_BIG) {
          err = page_retire_ex(mc, node_largedata_pgno(leaf_node), nullptr, 0);
          if (unlikely(err != MDBX_SUCCESS))
            return err;
        }
      }
    }
  }

  return page_retire(mc, pgr.page);
}

static int cutoff_zikkurat(MDBX_cursor *begin, MDBX_cursor *end, intptr_t level, bool end_including) {
  ASSERT(begin->txn == end->txn && is_pointed(begin) && is_pointed(end));
  ASSERT(begin->clc == end->clc && begin->top == end->top);
  ASSERT(level >= 0 && level <= begin->top);
  ASSERT(level < end->top);

  if (end_including) {
    for (intptr_t i = level; i <= end->top; ++i)
      if (end->ki[i] < page_numkeys(end->pg[i]) - 1) {
        end_including = false;
        break;
      }
    if (end_including && inner_pointed(end)) {
      for (intptr_t i = 0; i <= end->subcur->cursor.top; ++i)
        if (end->subcur->cursor.ki[i] < page_numkeys(end->subcur->cursor.pg[i]) - 1) {
          end_including = false;
          break;
        }
    }
  }

#ifndef NDEBUG
  VERBOSE(">> %s", cursor_dump_stack(begin, "begin", false, false));
  VERBOSE(">> %s", cursor_dump_stack(end, end_including ? "end-including" : "end-excluding", false, false));
#endif /* NDEBUG */

  cursor_couple_t dozer;
  MDBX_cursor *const mc = cursor_clone_complete(end, &dozer);
  dozer.outer.next = mc->txn->cursors[cursor_dbi(mc)];
  mc->txn->cursors[cursor_dbi(mc)] = &dozer.outer;
  int err = MDBX_SUCCESS;
  do {
    cursor_cpstk(end, mc);
    mc->top = level;

    while (true) {
      if (mc->tree->items < 4)
        goto bailout;

#ifndef NDEBUG
      VERBOSE(">= %s", cursor_dump_stack(begin, "begin", false, false));
      VERBOSE(">= %s", cursor_dump_stack(end, end_including ? "end-including" : "end-excluding", false, false));
#endif /* NDEBUG */

      page_t *mp = mc->pg[level];
      const indx_t nkeys = page_numkeys(mp);
      if (nkeys < 3)
        /* пропускаем, чтобы избежать слияния страниц, так как при этом
         * может сильно изменится конфигурация дерева, что потребует перезапуска процедуры быстрого удаления */
        goto next;
      const intptr_t ki_begin = 1 /* Пропускаем начальные узлы на branch-страницах,
                                   * чтобы не заниматься слиянием и обновлением ключей в родительских страницах */
                                + ((mp != begin->pg[level]) ? 0 : begin->ki[level]);
      const intptr_t ki_end = (mp != end->pg[level]) ? nkeys : end->ki[level] + end_including;
      if (ki_begin >= ki_end)
        goto next;

      ASSERT((begin->flags & z_after_delete) == 0);
      err = cursor_touch(mc, nullptr, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        goto bailout;
      mp = mc->pg[mc->top];
      mc->ki[level] = ki_end;
      do {
        node_t *const node = page_node(mp, mc->ki[level] -= 1);
        cASSERT0(mc, is_branch(mp) && node_flags(node) == 0);
#ifndef NDEBUG
        VERBOSE("== %s", cursor_dump_stack(mc, "del-tree-node", false, false));
#endif /* NDEBUG */
        err = tree_cutoff_twig(mc, node_pgno(node), level + 1, mp->txnid, false);
        if (unlikely(err != MDBX_SUCCESS))
          goto bailout;
        node_del(mc, 0);
      } while (page_numkeys(mp) > 2 && mc->ki[level] > ki_begin);

      const indx_t dropped = nkeys - page_numkeys(mp);
      for (MDBX_cursor *m2 = dozer.outer.next; m2; m2 = m2->next) {
        MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
        if (m3->top < level || m3->pg[level] != mp || m3->ki[level] < mc->ki[level])
          continue;
        if (m3->ki[level] >= ki_end) {
          m3->ki[level] -= dropped;
          ASSERT(level != m3->top);
          /* if (level == m3->top && inner_pointed(m3))
            cursor_inner_refresh(m3, mp, m3->ki[level]); */
        } else {
          ASSERT(m3 != begin);
          ASSERT(m3 != end || end_including);
          end_including &= m3 != end;
          m3->top = level;
          if (mc->ki[level] < page_numkeys(mp))
            m3->ki[level] = mc->ki[level];
          else
            err = cursor_sibling_right(m3);
          if (unlikely(err != MDBX_SUCCESS) && err != MDBX_NOTFOUND)
            goto bailout;
          err = tree_deepen_edge(m3, (err == MDBX_NOTFOUND) ? Z_LAST : Z_FIRST);
          if (unlikely(err != MDBX_SUCCESS))
            goto bailout;
          if (inner_pointed(m3))
            cursor_inner_refresh(m3, m3->pg[m3->top], m3->ki[m3->top]);
          m3->flags |= z_after_delete;
        }
      }

#ifndef NDEBUG
      VERBOSE("<= %s", cursor_dump_stack(begin, "begin", false, false));
      VERBOSE("<= %s", cursor_dump_stack(end, end_including ? "end-including" : "end-excluding", false, false));
#endif /* NDEBUG */

    next:
      err = cursor_sibling_left(mc);
      if (unlikely(err == MDBX_NOTFOUND))
        break;
      if (unlikely(err != MDBX_SUCCESS))
        goto bailout;
      for (intptr_t i = 0; i <= level; ++i)
        if (mc->ki[i] < begin->ki[i])
          goto bailout;
    }
  } while (++level < begin->top);
  err = MDBX_SUCCESS;

bailout:
#ifndef NDEBUG
  VERBOSE(">> %s", cursor_dump_stack(begin, "begin", false, false));
  VERBOSE(">> %s", cursor_dump_stack(end, end_including ? "end-including" : "end-excluding", false, false));
#endif /* NDEBUG */
  mc->txn->cursors[cursor_dbi(mc)] = dozer.outer.next;
  if (is_inner(mc)) {
    *begin->tree = *mc->tree;
    *end->tree = *mc->tree;
  }
  if (likely(err == MDBX_SUCCESS) && (end->flags & z_after_delete))
    err = MDBX_RESULT_TRUE;
  return err;
}

static int cutoff_storey(MDBX_cursor *begin, MDBX_cursor *end, intptr_t level, bool end_including) {
  ASSERT(begin->txn == end->txn && is_pointed(begin) && is_pointed(end));
  ASSERT(begin->clc == end->clc && begin->top == end->top);
  ASSERT(level >= 0 && level <= begin->top);
  ASSERT(memcmp(begin->tree, end->tree, sizeof(tree_t)) == 0);
  cASSERT0(begin, (begin->flags & (z_after_delete | z_eof_soft | z_eof_hard)) == 0);

  int err;
  intptr_t top = end->top;
  if (!is_inner(end)) {
    err = cursor_touch(begin, nullptr, nullptr);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if (end->pg[top] != begin->pg[top]) {
      err = cursor_touch(end, nullptr, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }
    if (MDBX_ENABLE_BUNCHES_REMOVAL && level < top) {
      err = cutoff_zikkurat(begin, end, level, end_including);
      if (unlikely(err != MDBX_SUCCESS)) {
        if (unlikely(err != MDBX_RESULT_TRUE))
          return err;
        if (begin->flags & z_after_delete)
          return MDBX_SUCCESS;
        end_including = false;
      }
      cASSERT0(begin, is_filled(begin));
      ASSERT(tree_diff_level(begin, end) >= MDBX_RESULT_TRUE);
    }
    ASSERT(is_pointed(begin) && is_pointed(end));
    while (end->pg[top] != begin->pg[top] || end->ki[top] > begin->ki[top]) {
      if (!end_including) {
        err = outer_prev(end, nullptr, nullptr, MDBX_PREV_NODUP);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
        if (end->pg[top] == begin->pg[top] && end->ki[top] == begin->ki[top])
          break;
      }
      ASSERT(is_pointed(begin) && is_pointed(end));
      ASSERT(tree_diff_level(begin, end) >= 0);
      err = cutoff_leaf(end, MDBX_ALLDUPS);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      ASSERT(end->top == begin->top);
      top = end->top;
      end_including = false;
    }
    ASSERT(is_pointed(begin) && is_pointed(end));
    ASSERT(begin->tree->items > 1);
    ASSERT(tree_diff_level(begin, end) >= MDBX_RESULT_TRUE);
    return cutoff_leaf(begin, MDBX_ALLDUPS);
  }

  MDBX_cursor *outer = cursor_outer(end);
  ASSERT(begin->pg[0] == end->pg[0]);
  err = cursor_touch(outer, nullptr, nullptr);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  node_t *outer_node = page_node(outer->pg[outer->top], outer->ki[outer->top]);
  ASSERT(node_flags(outer_node) & N_DUP);
  if (node_flags(outer_node) & N_TREE) {
    err = cursor_touch(end, nullptr, nullptr);
    if (unlikely(err != MDBX_SUCCESS))
      return err;

    begin->tree->mod_txnid = end->tree->mod_txnid = end->txn->txnid;
    begin->tree->root = end->tree->root;
    memcpy(node_data(outer_node), end->tree, sizeof(tree_t));

    if (end->pg[top] != begin->pg[top]) {
      err = cursor_touch(begin, nullptr, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }
  } else {
#ifndef MDBX_EVENBUG20260405_FIX
    end->pg[0] = begin->pg[0];
#endif /* !MDBX_EVENBUG20260405_FIX */
  }
  ASSERT(memcmp(begin->tree, end->tree, sizeof(tree_t)) == 0);

  if (end->tree->items > 2) {
    const uint64_t dups = end->tree->items;
    if (MDBX_ENABLE_BUNCHES_REMOVAL && level < top) {
      err = cutoff_zikkurat(begin, end, level, end_including);
      if (unlikely(err != MDBX_SUCCESS)) {
        if (unlikely(err != MDBX_RESULT_TRUE))
          return err;
        end_including = false;
        if (begin->flags & z_after_delete)
          return MDBX_SUCCESS;
      }
      cASSERT0(begin, is_filled(begin));
      ASSERT(tree_diff_level(begin, end) >= MDBX_RESULT_TRUE);
    }
    ASSERT(is_pointed(begin) && is_pointed(end));
    while (end->pg[top] != begin->pg[top] || end->ki[top] > begin->ki[top]) {
      if (!end_including) {
        err = inner_prev(end, nullptr);
        if (unlikely(err != MDBX_SUCCESS))
          return err;
        if (end->pg[top] == begin->pg[top] && end->ki[top] == begin->ki[top])
          break;
      }
      ASSERT(is_pointed(begin) && is_pointed(end));
      ASSERT(tree_diff_level(begin, end) >= 0);
      err = cutoff_leaf(end, 0);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      ASSERT(end->top == begin->top);
      top = end->top;
      end_including = false;
      if (end->tree->items == 2)
        break;
    }
    ASSERT(is_pointed(begin) && is_pointed(end));
    ASSERT(end->tree->items > 1);
    outer->tree->items -= dups - end->tree->items;
    *begin->tree = *end->tree;
  }

  ASSERT(is_pointed(begin) && is_pointed(end));
  ASSERT(tree_diff_level(begin, end) >= MDBX_RESULT_TRUE);
  intptr_t left = end->ki[top] - begin->ki[top] + end_including;
  /* Алгоритмически такая проверка корректна. Однако, на практике возможны ситуации, когда end->pg[top] !=
   * begin->pg[top], когда на каждую листовую страницу помещается только по одному ключу, это последние два ключа в
   * под-дереве и один из них должен быть удалён. При этом проверки результатов tree_diff_level() ниже достаточно для
   * проверки корректности удаления. Поэтому проще отключить эту проверку, нежели усложнять условия.
   * ASSERT(end->pg[top] == begin->pg[top] && left >= 0); */
  outer = cursor_outer(begin);
  do {
    ASSERT(tree_diff_level(outer, cursor_outer(end)) >= MDBX_RESULT_TRUE ||
           tree_diff_level(begin, end) >= MDBX_RESULT_TRUE);
    err = cutoff_leaf(outer, 0);
    if (unlikely(err != MDBX_SUCCESS))
      break;
    end->pg[top] = begin->pg[top];
    *end->tree = *begin->tree;
  } while (--left > 0);
  return err;
}

#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */

int tree_curoff_range(MDBX_cursor *begin, MDBX_cursor *end, bool end_including) {
  cASSERT1(begin, cursor_is_tracked(begin));
  cASSERT1(end, cursor_is_tracked(end));
  ASSERT(begin->txn == end->txn && begin->clc == end->clc && begin->dbi_state == end->dbi_state &&
         is_inner(begin) == is_inner(end) && (begin->tree == end->tree || is_inner(begin)) && begin->top == end->top);

  if (unlikely(begin->clc != end->clc))
    return MDBX_PROBLEM;
  if (unlikely(!is_pointed(begin) || !is_pointed(end)))
    return MDBX_ENODATA;
  if (unlikely(begin->top != end->top))
    return MDBX_PROBLEM;

#ifndef NDEBUG
  VERBOSE(">> %s", cursor_dump_stack(begin, "begin", true, false));
  VERBOSE(">> %s", cursor_dump_stack(end, end_including ? "end-including" : "end-excluding", true, false));
#endif /* NDEBUG */

#if MDBX_ENABLE_BUNCHES_REMOVAL
  /* ищем ближайший к корню ярус с различием в позициях */
  const intptr_t level = tree_diff_level(begin, end);
  if (level < 0) {
    if (unlikely(level != MDBX_RESULT_TRUE))
      return MDBX_ENODATA;

    /* позиции основных курсоров совпадают, смотрим на вложенные dupsort-курсоры */
    if (!begin->subcur)
      /* вложенных dupsort-курсоров нет, нечего удалять */
      return end_including ? cutoff_leaf(begin, 0) : MDBX_SUCCESS;

    const intptr_t inner_level = tree_diff_level(&begin->subcur->cursor, &end->subcur->cursor);
    if (unlikely(inner_level < 0)) {
      if (unlikely(inner_level != MDBX_RESULT_TRUE))
        return MDBX_ENODATA;
      /* у вложенных dupsort-курсоров позиции тоже совпадают, нечего удалять */
      return end_including ? cutoff_leaf(begin, 0) : MDBX_SUCCESS;
    }

    if (end_including && cursor_on_first(&begin->subcur->cursor) && cursor_on_last(&end->subcur->cursor))
      /* удаляем целиком dupsort-узел */
      return cutoff_leaf(begin, MDBX_ALLDUPS);
    else
      /* удаляем диапазон во вложенном dupsort-дереве */
      return cutoff_storey(&begin->subcur->cursor, &end->subcur->cursor, inner_level, end_including);
  }

  if (end_including && cursor_on_first(begin) && cursor_on_last(end)) {
    if (is_inner(begin))
      /* специфический случай когда функция была вызвана для вложенного dupsort-дерева */
      return cutoff_leaf(cursor_outer(begin), MDBX_ALLDUPS);
    else if (!begin->subcur || (cursor_on_first(&begin->subcur->cursor) && cursor_on_last(&end->subcur->cursor)))
      /* удаляем дерево целиком */
      return tbl_purge(begin);
  }

  intptr_t cmp = -1;
  /* coverity[ASSERT_SIDE_EFFECT] */
  ASSERT((cmp = cursor_cmp(begin, end)) < 0 || (cmp == 0 && end_including));
  if (begin->subcur) {
    cursor_couple_t inner_trimmer;
    if (is_pointed(&end->subcur->cursor) && (!end_including || !cursor_on_last(&end->subcur->cursor))) {
      /* в конце подлежащего удалению диапазона находится часть dupsort-куста,
       * удаляем начало этого вложенного dupsort-дерева между end и предыдущим узлом основного дерева. */
      MDBX_cursor *const tail = &end->subcur->cursor;
      MDBX_cursor *const head = &cursor_clone_complete(end, &inner_trimmer)->subcur->cursor;
      int err = inner_first(head, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return err;

      const intptr_t inner_level = tree_diff_level(head, tail);
      ASSERT(inner_level >= MDBX_RESULT_TRUE);
      if (inner_level >= 0) {
        inner_trimmer.outer.next = head->txn->cursors[cursor_dbi(head)];
        head->txn->cursors[cursor_dbi(head)] = &inner_trimmer.outer;
        err = cutoff_storey(head, tail, inner_level, end_including);
        head->txn->cursors[cursor_dbi(head)] = inner_trimmer.outer.next;
      } else if (end_including) {
        err = cutoff_leaf(end, 0);
      }
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      cASSERT0(end, cursor_on_first(tail));
      err = outer_prev(end, nullptr, nullptr, MDBX_PREV_NODUP);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      cmp = cursor_cmp(begin, end);
      end_including = cmp < 0;
    }

    if (!cursor_on_first(&begin->subcur->cursor)) {
      /* в начале подлежащего удалению диапазона находится часть dupsort-куста,
       * удаляем конец этого вложенного dupsort-дерева между begin и следующим узлом основного дерева. */
      MDBX_cursor *const head = &begin->subcur->cursor;
      MDBX_cursor *const tail = &cursor_clone_complete(begin, &inner_trimmer)->subcur->cursor;
      int err = inner_last(tail, nullptr);
      if (unlikely(err != MDBX_SUCCESS))
        return err;

      const intptr_t inner_level = tree_diff_level(head, tail);
      ASSERT(inner_level >= MDBX_RESULT_TRUE);
      if (inner_level >= 0) {
        inner_trimmer.outer.next = tail->txn->cursors[cursor_dbi(tail)];
        tail->txn->cursors[cursor_dbi(tail)] = &inner_trimmer.outer;
        err = cutoff_storey(head, tail, inner_level, true);
        tail->txn->cursors[cursor_dbi(tail)] = inner_trimmer.outer.next;
      } else {
        err = cutoff_leaf(begin, 0);
      }
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      cASSERT0(begin, cursor_on_last(head));
      err = outer_next(begin, nullptr, nullptr, MDBX_NEXT_NODUP);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      cASSERT0(begin, cursor_on_first(head));
      cmp = cursor_cmp(begin, end);
      if (cmp >= end_including)
        return MDBX_SUCCESS;
    }
  }

  return cutoff_storey(begin, end, level, end_including);

#else /* MDBX_ENABLE_BUNCHES_REMOVAL */

  intptr_t cmp = tree_cmp(begin, end);
  if (unlikely(cmp > 0))
    return MDBX_ENODATA;
  if (cmp == 0 && inner_pointed(begin) && unlikely(tree_cmp(&begin->subcur->cursor, &end->subcur->cursor) > 0))
    return MDBX_ENODATA;

  int err;
  while (cmp < 0) {
    err = cutoff_leaf(begin, (begin->subcur && cursor_on_first(&begin->subcur->cursor)) ? MDBX_ALLDUPS : 0);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    if ((begin->flags & (z_eof_soft | z_eof_hard)) != 0 ||
        (inner_pointed(begin) && (begin->subcur->cursor.flags & (z_eof_soft | z_eof_hard))) != 0) {
      err = outer_next(begin, nullptr, nullptr, MDBX_NEXT);
      if (unlikely(err != MDBX_SUCCESS) && err != MDBX_NOTFOUND)
        return err;
    }
    cmp = tree_cmp(begin, end);
  }

  if (cmp == 0 && inner_pointed(begin)) {
    cmp = tree_cmp(&begin->subcur->cursor, &end->subcur->cursor);
    while (cmp < 0) {
      err = cutoff_leaf(begin, 0);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
      cmp = tree_cmp(begin, end);
      if (cmp == 0 && inner_pointed(begin))
        cmp = tree_cmp(&begin->subcur->cursor, &end->subcur->cursor);
    }
  }

  return (end_including && cmp == 0 && is_filled(end)) ? cutoff_leaf(end, 0) : MDBX_SUCCESS;

#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
}

void recalculate_merge_thresholds(MDBX_env *env) {
  const size_t whole_page_space = page_space(env);
  env->merge_threshold = (uint16_t)(whole_page_space - (whole_page_space * env->options.merge_threshold_dot16 >> 16));
  eASSERT0(env, env->merge_threshold >= whole_page_space / 2u && env->merge_threshold <= whole_page_space * 63u / 64u);
}

int tree_drop(MDBX_cursor *mc) {
#if MDBX_ENABLE_BUNCHES_REMOVAL
  int rc = tree_cutoff_twig(mc, mc->tree->root, 1, tbl_root_txnid(mc->txn, cursor_dbi(mc)), true);
#else
  const bool may_have_subtrees = !is_inner(mc) && (cursor_is_main(mc) || (mc->tree->flags & MDBX_DUPSORT));
  int rc = tree_search(mc, nullptr, Z_FIRST);
  if (likely(rc == MDBX_SUCCESS)) {
    /* DUPSORT sub-DBs have no large-pages/tables. Omit scanning leaves.
     * This also avoids any P_DUPFIX pages, which have no nodes.
     * Also if the DB doesn't have sub-DBs and has no large/overflow
     * pages, omit scanning leaves. */
    if (!(may_have_subtrees | mc->tree->large_pages))
      cursor_pop(mc);

    rc = pnl_need(&mc->txn->wr.retired_pages,
                  (size_t)mc->tree->branch_pages + (size_t)mc->tree->leaf_pages + (size_t)mc->tree->large_pages);
    if (unlikely(rc != MDBX_SUCCESS))
      goto bailout;

    page_t *stack[CURSOR_STACK_SIZE];
    for (intptr_t i = 0; i <= mc->top; ++i)
      stack[i] = mc->pg[i];

    while (mc->top >= 0) {
      page_t *const mp = mc->pg[mc->top];
      const size_t nkeys = page_numkeys(mp);
      if (is_leaf(mp)) {
        cASSERT0(mc, mc->top + 1 == mc->tree->height);
        for (size_t i = 0; i < nkeys; i++) {
          node_t *node = page_node(mp, i);
          if (node_flags(node) & N_BIG) {
            rc = page_retire_ex(mc, node_largedata_pgno(node), nullptr, 0);
            if (unlikely(rc != MDBX_SUCCESS))
              goto bailout;
            if (!(may_have_subtrees | mc->tree->large_pages))
              goto popup;
          } else if (node_flags(node) & N_TREE) {
            if (unlikely((node_flags(node) & N_DUP) == 0)) {
              rc = /* disallowing implicit table deletion */ MDBX_INCOMPATIBLE;
              goto bailout;
            }
            rc = cursor_dupsort_setup(mc, node, mp);
            if (unlikely(rc != MDBX_SUCCESS))
              goto bailout;
            rc = tree_drop(&mc->subcur->cursor);
            if (unlikely(rc != MDBX_SUCCESS))
              goto bailout;
          }
        }
      } else {
        cASSERT0(mc, mc->top + 1 < mc->tree->height);
        mc->checking |= z_retiring;
        const unsigned pagetype = (is_frozen(mc->txn, mp) ? P_FROZEN : 0) +
                                  ((mc->top + 2 == mc->tree->height) ? (mc->checking & (P_LEAF | P_DUPFIX)) : P_BRANCH);
        for (size_t i = 0; i < nkeys; i++) {
          node_t *node = page_node(mp, i);
          cASSERT0(mc, (node_flags(node) & (N_BIG | N_TREE | N_DUP)) == 0);
          const pgno_t pgno = node_pgno(node);
          rc = page_retire_ex(mc, pgno, nullptr, pagetype);
          if (unlikely(rc != MDBX_SUCCESS))
            goto bailout;
        }
        mc->checking -= z_retiring;
      }
      if (!mc->top)
        break;
      cASSERT0(mc, nkeys > 0);
      mc->ki[mc->top] = (indx_t)nkeys;
      rc = cursor_sibling_right(mc);
      if (unlikely(rc != MDBX_SUCCESS)) {
        if (unlikely(rc != MDBX_NOTFOUND))
          goto bailout;
        /* no more siblings, go back to beginning of previous level. */
      popup:
        cursor_pop(mc);
        mc->ki[0] = 0;
        for (intptr_t i = 1; i <= mc->top; i++) {
          mc->pg[i] = stack[i];
          mc->ki[i] = 0;
        }
      }
    }
    rc = page_retire(mc, mc->pg[0]);
  }

bailout:
#endif /* MDBX_ENABLE_BUNCHES_REMOVAL */
  be_poor(mc);
  return rc;
}

static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
  int rc;
  DKBUF_DEBUG;

  page_t *psrc = csrc->pg[csrc->top];
  page_t *pdst = cdst->pg[cdst->top];
  cASSERT0(csrc, page_type(psrc) == page_type(pdst));
  cASSERT0(csrc, csrc->tree == cdst->tree);
  cASSERT0(csrc, csrc->top == cdst->top);
  if (unlikely(page_type(psrc) != page_type(pdst))) {
  bailout:
    ERROR("Wrong or mismatch pages's types (src %d, dst %d) to move node", page_type(psrc), page_type(pdst));
    csrc->txn->flags |= MDBX_TXN_ERROR;
    return MDBX_PROBLEM;
  }

  MDBX_val key4move;
  switch (page_type(psrc)) {
  case P_BRANCH: {
    const node_t *srcnode = page_node(psrc, csrc->ki[csrc->top]);
    cASSERT0(csrc, node_flags(srcnode) == 0);
    const pgno_t srcpg = node_pgno(srcnode);
    key4move.iov_len = node_ks(srcnode);
    key4move.iov_base = node_key(srcnode);

    if (csrc->ki[csrc->top] == 0) {
      const int8_t top = csrc->top;
      cASSERT0(csrc, top >= 0);
      /* must find the lowest key below src */
      rc = tree_deepen_lowest(csrc);
      page_t *lowest_page = csrc->pg[csrc->top];
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      cASSERT0(csrc, is_leaf(lowest_page));
      if (unlikely(!is_leaf(lowest_page)))
        goto bailout;
      if (is_dupfix_leaf(lowest_page))
        key4move = page_dupfix_key(lowest_page, 0, csrc->tree->dupfix_size);
      else {
        const node_t *lowest_node = page_node(lowest_page, 0);
        key4move.iov_len = node_ks(lowest_node);
        key4move.iov_base = node_key(lowest_node);
      }

      /* restore cursor after mdbx_page_search_lowest() */
      csrc->top = top;
      csrc->ki[csrc->top] = 0;

      /* paranoia */
      cASSERT0(csrc, psrc == csrc->pg[csrc->top]);
      cASSERT0(csrc, is_branch(psrc));
      if (unlikely(!is_branch(psrc)))
        goto bailout;
    }

    if (cdst->ki[cdst->top] == 0) {
      cursor_couple_t couple;
      MDBX_cursor *const mn = cursor_clone_slightly(cdst, &couple);
      const int8_t top = cdst->top;
      cASSERT0(csrc, top >= 0);

      /* must find the lowest key below dst */
      rc = tree_deepen_lowest(mn);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      page_t *const lowest_page = mn->pg[mn->top];
      cASSERT0(cdst, is_leaf(lowest_page));
      if (unlikely(!is_leaf(lowest_page)))
        goto bailout;
      MDBX_val key;
      if (is_dupfix_leaf(lowest_page))
        key = page_dupfix_key(lowest_page, 0, mn->tree->dupfix_size);
      else {
        node_t *lowest_node = page_node(lowest_page, 0);
        key.iov_len = node_ks(lowest_node);
        key.iov_base = node_key(lowest_node);
      }

      /* restore cursor after mdbx_page_search_lowest() */
      mn->top = top;
      mn->ki[mn->top] = 0;

      const intptr_t delta = EVEN_CEIL(key.iov_len) - EVEN_CEIL(node_ks(page_node(mn->pg[mn->top], 0)));
      const intptr_t needed = branch_size(cdst->txn->env, &key4move) + delta;
      const intptr_t have = page_room(pdst);
      if (unlikely(needed > have))
        return MDBX_RESULT_TRUE;

      if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst))))
        return rc;
      psrc = csrc->pg[csrc->top];
      pdst = cdst->pg[cdst->top];

      couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
      mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
      rc = tree_propagate_key(mn, &key);
      mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
    } else {
      const size_t needed = branch_size(cdst->txn->env, &key4move);
      const size_t have = page_room(pdst);
      if (unlikely(needed > have))
        return MDBX_RESULT_TRUE;

      if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst))))
        return rc;
      psrc = csrc->pg[csrc->top];
      pdst = cdst->pg[cdst->top];
    }

    DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "branch", csrc->ki[csrc->top],
          DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno);
    /* Add the node to the destination page. */
    rc = node_add_branch(cdst, cdst->ki[cdst->top], &key4move, srcpg);
  } break;

  case P_LEAF: {
    /* Mark src and dst as dirty. */
    if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst))))
      return rc;
    psrc = csrc->pg[csrc->top];
    pdst = cdst->pg[cdst->top];
    const node_t *srcnode = page_node(psrc, csrc->ki[csrc->top]);
    MDBX_val data;
    data.iov_len = node_ds(srcnode);
    data.iov_base = node_data(srcnode);
    key4move.iov_len = node_ks(srcnode);
    key4move.iov_base = node_key(srcnode);
    DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf", csrc->ki[csrc->top],
          DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno);
    /* Add the node to the destination page. */
    rc = node_add_leaf(cdst, cdst->ki[cdst->top], &key4move, &data, node_flags(srcnode));
  } break;

  case P_LEAF | P_DUPFIX: {
    /* Mark src and dst as dirty. */
    if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst))))
      return rc;
    psrc = csrc->pg[csrc->top];
    pdst = cdst->pg[cdst->top];
    key4move = page_dupfix_key(psrc, csrc->ki[csrc->top], csrc->tree->dupfix_size);
    DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf2", csrc->ki[csrc->top],
          DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno);
    /* Add the node to the destination page. */
    rc = node_add_dupfix(cdst, cdst->ki[cdst->top], &key4move);
  } break;

  default:
    ASSERT(false);
    goto bailout;
  }

  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  /* Delete the node from the source page. */
  node_del(csrc, key4move.iov_len);

  cASSERT0(csrc, psrc == csrc->pg[csrc->top]);
  cASSERT0(cdst, pdst == cdst->pg[cdst->top]);
  cASSERT0(csrc, page_type(psrc) == page_type(pdst));

  /* csrc курсор тут всегда временный, на стеке внутри tree_rebalance(),
   * и его нет необходимости корректировать. */
  {
    /* Adjust other cursors pointing to mp */
    MDBX_cursor *m2, *m3;
    const size_t dbi = cursor_dbi(csrc);
    cASSERT0(csrc, csrc->top == cdst->top);
    if (fromleft) {
      /* Перемещаем с левой страницы нв правую, нужно сдвинуть ki на +1 */
      for (m2 = csrc->txn->cursors[dbi]; m2; m2 = m2->next) {
        m3 = (csrc->flags & z_inner) ? &m2->subcur->cursor : m2;
        if (!is_related(csrc, m3))
          continue;

        if (m3 != cdst && m3->pg[csrc->top] == pdst && m3->ki[csrc->top] >= cdst->ki[csrc->top]) {
          m3->ki[csrc->top] += 1;
        }

        if (/* m3 != csrc && */ m3->pg[csrc->top] == psrc && m3->ki[csrc->top] == csrc->ki[csrc->top]) {
          m3->pg[csrc->top] = pdst;
          m3->ki[csrc->top] = cdst->ki[cdst->top];
          cASSERT0(csrc, csrc->top > 0);
          m3->ki[csrc->top - 1] += 1;
        }

        if (is_leaf(psrc) && inner_pointed(m3)) {
          cASSERT0(csrc, csrc->top == m3->top);
          size_t nkeys = page_numkeys(m3->pg[csrc->top]);
          if (likely(nkeys > m3->ki[csrc->top]))
            cursor_inner_refresh(m3, m3->pg[csrc->top], m3->ki[csrc->top]);
        }
      }
    } else {
      /* Перемещаем с правой страницы на левую, нужно сдвинуть ki на -1 */
      for (m2 = csrc->txn->cursors[dbi]; m2; m2 = m2->next) {
        m3 = (csrc->flags & z_inner) ? &m2->subcur->cursor : m2;
        if (!is_related(csrc, m3))
          continue;
        if (m3->pg[csrc->top] == psrc) {
          if (!m3->ki[csrc->top]) {
            m3->pg[csrc->top] = pdst;
            m3->ki[csrc->top] = cdst->ki[cdst->top];
            cASSERT0(csrc, csrc->top > 0 && m3->ki[csrc->top - 1] > 0);
            m3->ki[csrc->top - 1] -= 1;
          } else
            m3->ki[csrc->top] -= 1;

          if (is_leaf(psrc) && inner_pointed(m3)) {
            cASSERT0(csrc, csrc->top == m3->top);
            size_t nkeys = page_numkeys(m3->pg[csrc->top]);
            if (likely(nkeys > m3->ki[csrc->top]))
              cursor_inner_refresh(m3, m3->pg[csrc->top], m3->ki[csrc->top]);
          }
        }
      }
    }
  }

  /* Update the parent separators. */
  if (csrc->ki[csrc->top] == 0) {
    cASSERT0(csrc, csrc->top > 0);
    if (csrc->ki[csrc->top - 1] != 0) {
      MDBX_val key;
      if (is_dupfix_leaf(psrc))
        key = page_dupfix_key(psrc, 0, csrc->tree->dupfix_size);
      else {
        node_t *srcnode = page_node(psrc, 0);
        key.iov_len = node_ks(srcnode);
        key.iov_base = node_key(srcnode);
      }
      DEBUG("update separator for source page %" PRIaPGNO " to [%s]", psrc->pgno, DKEY_DEBUG(&key));

      cursor_couple_t couple;
      MDBX_cursor *const mn = cursor_clone_slightly(csrc, &couple);
      cASSERT0(csrc, mn->top > 0);
      mn->top -= 1;

      couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
      mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
      rc = tree_propagate_key(mn, &key);
      mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
    }
    if (is_branch(psrc)) {
      const MDBX_val nullkey = {0, 0};
      const indx_t ix = csrc->ki[csrc->top];
      csrc->ki[csrc->top] = 0;
      rc = tree_propagate_key(csrc, &nullkey);
      csrc->ki[csrc->top] = ix;
      cASSERT0(csrc, rc == MDBX_SUCCESS);
    }
  }

  if (cdst->ki[cdst->top] == 0) {
    cASSERT0(cdst, cdst->top > 0);
    if (cdst->ki[cdst->top - 1] != 0) {
      MDBX_val key;
      if (is_dupfix_leaf(pdst))
        key = page_dupfix_key(pdst, 0, cdst->tree->dupfix_size);
      else {
        node_t *srcnode = page_node(pdst, 0);
        key.iov_len = node_ks(srcnode);
        key.iov_base = node_key(srcnode);
      }
      DEBUG("update separator for destination page %" PRIaPGNO " to [%s]", pdst->pgno, DKEY_DEBUG(&key));
      cursor_couple_t couple;
      MDBX_cursor *const mn = cursor_clone_slightly(cdst, &couple);
      cASSERT0(cdst, mn->top > 0);
      mn->top -= 1;

      couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
      mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
      rc = tree_propagate_key(mn, &key);
      mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
    }
    if (is_branch(pdst)) {
      const MDBX_val nullkey = {0, 0};
      const indx_t ix = cdst->ki[cdst->top];
      cdst->ki[cdst->top] = 0;
      rc = tree_propagate_key(cdst, &nullkey);
      cdst->ki[cdst->top] = ix;
      cASSERT0(cdst, rc == MDBX_SUCCESS);
    }
  }

  return MDBX_SUCCESS;
}

static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
  MDBX_val key;
  int rc;

  cASSERT0(csrc, csrc != cdst);
  cASSERT1(csrc, cursor_is_tracked(csrc));
  cASSERT1(cdst, cursor_is_tracked(cdst));
  const page_t *const psrc = csrc->pg[csrc->top];
  page_t *pdst = cdst->pg[cdst->top];
  DEBUG("merging page %" PRIaPGNO " into %" PRIaPGNO, psrc->pgno, pdst->pgno);

  cASSERT0(csrc, page_type(psrc) == page_type(pdst));
  cASSERT0(csrc, csrc->clc == cdst->clc && csrc->tree == cdst->tree);
  cASSERT0(csrc, csrc->top > 0); /* can't merge root page */
  cASSERT0(cdst, cdst->top > 0);
  cASSERT0(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1]));
  cASSERT0(csrc, csrc->top + 1 < csrc->tree->height || is_leaf(csrc->pg[csrc->tree->height - 1]));
  cASSERT0(cdst, cursor_dbi(csrc) == FREE_DBI || csrc->txn->env->options.prefer_waf_insteadof_balance ||
                     page_room(pdst) >= page_used(cdst->txn->env, psrc));
  const int pagetype = page_type(psrc);

  /* Move all nodes from src to dst */
  const size_t dst_nkeys = page_numkeys(pdst);
  const size_t src_nkeys = page_numkeys(psrc);
  cASSERT0(cdst, dst_nkeys + src_nkeys >= (is_leaf(psrc) ? 1u : 2u));
  if (likely(src_nkeys)) {
    size_t ii = dst_nkeys;
    if (unlikely(pagetype & P_DUPFIX)) {
      /* Mark dst as dirty. */
      rc = page_touch(cdst);
      cASSERT0(cdst, rc != MDBX_RESULT_TRUE);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;

      key.iov_len = csrc->tree->dupfix_size;
      key.iov_base = page2payload(psrc);
      size_t i = 0;
      do {
        rc = node_add_dupfix(cdst, ii++, &key);
        cASSERT0(cdst, rc != MDBX_RESULT_TRUE);
        if (unlikely(rc != MDBX_SUCCESS))
          return rc;
        key.iov_base = ptr_disp(key.iov_base, key.iov_len);
      } while (++i != src_nkeys);
    } else {
      node_t *srcnode = page_node(psrc, 0);
      key.iov_len = node_ks(srcnode);
      key.iov_base = node_key(srcnode);
      if (pagetype & P_BRANCH) {
        cursor_couple_t couple;
        MDBX_cursor *const mn = cursor_clone_slightly(csrc, &couple);

        /* must find the lowest key below src */
        rc = tree_deepen_lowest(mn);
        cASSERT0(csrc, rc != MDBX_RESULT_TRUE);
        if (unlikely(rc != MDBX_SUCCESS))
          return rc;

        const page_t *mp = mn->pg[mn->top];
        if (likely(!is_dupfix_leaf(mp))) {
          cASSERT0(mn, is_leaf(mp));
          const node_t *lowest = page_node(mp, 0);
          key.iov_len = node_ks(lowest);
          key.iov_base = node_key(lowest);
        } else {
          cASSERT0(mn, mn->top > csrc->top);
          key = page_dupfix_key(mp, mn->ki[mn->top], csrc->tree->dupfix_size);
        }
        cASSERT0(mn, key.iov_len >= csrc->clc->k.lmin);
        cASSERT0(mn, key.iov_len <= csrc->clc->k.lmax);

        const size_t dst_room = page_room(pdst);
        const size_t src_used = page_used(cdst->txn->env, psrc);
        const size_t space_needed = src_used - node_ks(srcnode) + key.iov_len;
        if (unlikely(space_needed > dst_room))
          return MDBX_RESULT_TRUE;
      }

      /* Mark dst as dirty. */
      rc = page_touch(cdst);
      cASSERT0(cdst, rc != MDBX_RESULT_TRUE);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;

      size_t i = 0;
      while (true) {
        if (pagetype & P_LEAF) {
          MDBX_val data;
          data.iov_len = node_ds(srcnode);
          data.iov_base = node_data(srcnode);
          rc = node_add_leaf(cdst, ii++, &key, &data, node_flags(srcnode));
        } else {
          cASSERT0(csrc, node_flags(srcnode) == 0);
          rc = node_add_branch(cdst, ii++, &key, node_pgno(srcnode));
        }
        cASSERT0(cdst, rc != MDBX_RESULT_TRUE);
        if (unlikely(rc != MDBX_SUCCESS))
          return rc;

        if (++i == src_nkeys)
          break;
        srcnode = page_node(psrc, i);
        key.iov_len = node_ks(srcnode);
        key.iov_base = node_key(srcnode);
      }
    }

    pdst = cdst->pg[cdst->top];
    DEBUG("dst page %" PRIaPGNO " now has %zu keys (%u.%u%% filled)", pdst->pgno, page_numkeys(pdst),
          page_fill_percentum_x10(cdst->txn->env, pdst) / 10, page_fill_percentum_x10(cdst->txn->env, pdst) % 10);

    cASSERT0(csrc, psrc == csrc->pg[csrc->top]);
    cASSERT0(cdst, pdst == cdst->pg[cdst->top]);
  }

  /* Unlink the src page from parent and add to free list. */
  csrc->top -= 1;
  node_del(csrc, 0);
  if (csrc->ki[csrc->top] == 0) {
    const MDBX_val nullkey = {0, 0};
    rc = tree_propagate_key(csrc, &nullkey);
    cASSERT0(csrc, rc != MDBX_RESULT_TRUE);
    if (unlikely(rc != MDBX_SUCCESS)) {
      csrc->top += 1;
      return rc;
    }
  }
  csrc->top += 1;

  cASSERT0(csrc, psrc == csrc->pg[csrc->top]);
  cASSERT0(cdst, pdst == cdst->pg[cdst->top]);

  {
    /* Adjust other cursors pointing to mp */
    MDBX_cursor *m2, *m3;
    const size_t dbi = cursor_dbi(csrc);
    for (m2 = csrc->txn->cursors[dbi]; m2; m2 = m2->next) {
      m3 = (csrc->flags & z_inner) ? &m2->subcur->cursor : m2;
      if (!is_related(csrc, m3))
        continue;
      if (m3->pg[csrc->top] == psrc) {
        m3->pg[csrc->top] = pdst;
        m3->ki[csrc->top] += (indx_t)dst_nkeys;
        m3->ki[csrc->top - 1] = cdst->ki[csrc->top - 1];
      } else if (m3->pg[csrc->top - 1] == csrc->pg[csrc->top - 1] && m3->ki[csrc->top - 1] > csrc->ki[csrc->top - 1]) {
        cASSERT0(m3, m3->ki[csrc->top - 1] > 0 && m3->ki[csrc->top - 1] <= page_numkeys(m3->pg[csrc->top - 1]));
        m3->ki[csrc->top - 1] -= 1;
      }

      if (is_leaf(psrc) && inner_pointed(m3)) {
        cASSERT0(csrc, csrc->top == m3->top);
        size_t nkeys = page_numkeys(m3->pg[csrc->top]);
        if (likely(nkeys > m3->ki[csrc->top]))
          cursor_inner_refresh(m3, m3->pg[csrc->top], m3->ki[csrc->top]);
      }
    }
  }

  rc = page_retire(csrc, (page_t *)psrc);
  cASSERT0(csrc, rc != MDBX_RESULT_TRUE);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  cASSERT0(cdst, cdst->tree->items > 0);
  cASSERT0(cdst, cdst->top + 1 <= cdst->tree->height);
  cASSERT0(cdst, cdst->top > 0);
  page_t *const top_page = cdst->pg[cdst->top];
  const indx_t top_indx = cdst->ki[cdst->top];
  const uint16_t save_height = cdst->tree->height;
  const int save_top = cdst->top;
  cursor_pop(cdst);
  rc = tree_rebalance(cdst);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  cASSERT0(cdst, cdst->tree->items > 0);
  cASSERT0(cdst, cdst->top + 1 <= cdst->tree->height);
  if (MDBX_ENABLE_PGOP_STAT)
    cdst->txn->env->lck->pgops.merge.weak += 1;

  if (is_leaf(cdst->pg[cdst->top])) {
    /* LY: don't touch cursor if top-page is a LEAF */
    cASSERT0(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
    return MDBX_SUCCESS;
  }

  cASSERT0(cdst, page_numkeys(top_page) == dst_nkeys + src_nkeys);
  const int new_top = save_top - save_height + cdst->tree->height;
  if (unlikely(pagetype != page_type(top_page))) {
    /* LY: LEAF-page becomes BRANCH, unable restore cursor's stack */
    ERROR("unexpected top-page type 0x%x, expect 0x%x", page_type(top_page), pagetype);
    goto bailout;
  }

  if (top_page == cdst->pg[cdst->top]) {
    /* LY: don't touch cursor if prev top-page already on the top */
    cASSERT0(cdst, cdst->ki[cdst->top] == top_indx);
    cASSERT0(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
    return MDBX_SUCCESS;
  }

  if (unlikely(new_top < 0 || new_top >= cdst->tree->height)) {
    /* LY: out of range, unable restore cursor's stack */
    ERROR("cursor top-new %i is out of range %u..%u (top-before %i, height-before %i, height-new %i)", new_top, 0,
          cdst->tree->height, save_top, save_height, cdst->tree->height);
    goto bailout;
  }

  if (top_page == cdst->pg[new_top]) {
    cASSERT0(cdst, cdst->ki[new_top] == top_indx);
    /* LY: restore cursor stack */
    cdst->top = (int8_t)new_top;
    cASSERT0(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1]));
    cASSERT0(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
    return MDBX_SUCCESS;
  }

  if (save_height > cdst->tree->height && cdst->pg[save_top] == top_page && cdst->ki[save_top] == top_indx) {
    /* LY: restore cursor stack */
    cdst->pg[new_top] = top_page;
    cdst->ki[new_top] = top_indx;
#if MDBX_DEBUG > 0
    cdst->pg[new_top + 1] = nullptr;
    cdst->ki[new_top + 1] = INT16_MAX;
#endif /* MDBX_DEBUG */
    cdst->top = (int8_t)new_top;
    cASSERT0(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1]));
    cASSERT0(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
    return MDBX_SUCCESS;
  }

bailout:
  ERROR("unable restore %scursor stack after merge; "
        " new: height %i top %i top-idx %i top-page %p;"
        " before: height %i top %i, top-indx %i top-page %p",
        is_inner(cdst) ? "sub-" : "", cdst->tree->height, new_top, cdst->ki[save_top],
        __Wpedantic_format_voidptr(cdst->pg[save_top]), save_height, save_top, top_indx,
        __Wpedantic_format_voidptr(top_page));
  be_poor(cdst);
  return MDBX_CURSOR_FULL;
}

int tree_rebalance(MDBX_cursor *mc) {
  cASSERT1(mc, cursor_is_tracked(mc));
  cASSERT0(mc, mc->top >= 0);
  cASSERT0(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1]));
  const page_t *const tp = mc->pg[mc->top];
  const uint8_t pagetype = page_type(tp);

  STATIC_ASSERT(P_BRANCH == 1);
  const size_t minkeys = (pagetype & P_BRANCH) + (size_t)1;

  /* Pages emptier than this are candidates for merging. */
  size_t room_threshold = mc->txn->env->merge_threshold;
  bool minimize_waf = mc->txn->env->options.prefer_waf_insteadof_balance;
  if (unlikely(mc->tree == &mc->txn->dbs[FREE_DBI])) {
    /* В случае GC всегда минимизируем WAF, а рыхлые страницы объединяем только при наличии запаса в gc_stockpile().
     * Это позволяет уменьшить WAF и избавиться от лишних действий/циклов как при переработке GC,
     * так и при возврате неиспользованных страниц. Сбалансированность b-tree при этом почти не деградирует,
     * ибо добавление/удаление/обновление запиcей происходит почти всегда только по краям. */
    minimize_waf = true;
    room_threshold = page_space(mc->txn->env);
    if (gc_stockpile(mc->txn) > (size_t)mc->tree->height + mc->tree->height)
      room_threshold >>= 1;
  }

  const size_t numkeys = page_numkeys(tp);
  const size_t room = page_room(tp);
  DEBUG("rebalancing %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)",
        is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10,
        page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room);
  cASSERT0(mc, is_modifable(mc->txn, tp));

  if (unlikely(numkeys < minkeys)) {
    DEBUG("page %" PRIaPGNO " must be merged due keys < %zu threshold", tp->pgno, minkeys);
  } else if (unlikely(room > room_threshold)) {
    DEBUG("page %" PRIaPGNO " should be merged due room %zu > %zu threshold", tp->pgno, room, room_threshold);
  } else {
    DEBUG("no need to rebalance page %" PRIaPGNO ", room %zu < %zu threshold", tp->pgno, room, room_threshold);
    cASSERT0(mc, mc->tree->items > 0);
    return MDBX_SUCCESS;
  }

  int rc;
  if (mc->top == 0) {
    page_t *const mp = mc->pg[0];
    const size_t nkeys = page_numkeys(mp);
    cASSERT0(mc, (mc->tree->items == 0) == (nkeys == 0));
    if (nkeys == 0) {
      DEBUG("%s", "tree is completely empty");
      cASSERT0(mc, is_leaf(mp));
      cASSERT0(mc, (*cursor_dbi_state(mc) & DBI_DIRTY) != 0);
      cASSERT0(mc, mc->tree->branch_pages == 0 && mc->tree->large_pages == 0 && mc->tree->leaf_pages == 1);
      /* Adjust cursors pointing to mp */
      for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
        MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
        if (!is_poor(m3) && m3->pg[0] == mp) {
          be_poor(m3);
          m3->flags |= z_after_delete;
        }
      }
      if (is_subpage(mp)) {
        return MDBX_SUCCESS;
      } else {
        mc->tree->root = P_INVALID;
        mc->tree->height = 0;
        return page_retire(mc, mp);
      }
    }
    if (is_subpage(mp)) {
      DEBUG("%s", "Can't rebalance a subpage, ignoring");
      cASSERT0(mc, is_leaf(tp));
      return MDBX_SUCCESS;
    }
    if (is_branch(mp) && nkeys == 1) {
      DEBUG("%s", "collapsing root page!");
      mc->tree->root = node_pgno(page_node(mp, 0));
      rc = page_get(mc, mc->tree->root, &mc->pg[0], mp->txnid);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      mc->tree->height--;
      mc->ki[0] = mc->ki[1];
      for (intptr_t i = 1; i < mc->tree->height; i++) {
        mc->pg[i] = mc->pg[i + 1];
        mc->ki[i] = mc->ki[i + 1];
      }

      /* Adjust other cursors pointing to mp */
      for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
        MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
        if (is_related(mc, m3) && m3->pg[0] == mp) {
          for (intptr_t i = 0; i < mc->tree->height; i++) {
            m3->pg[i] = m3->pg[i + 1];
            m3->ki[i] = m3->ki[i + 1];
          }
          m3->top -= 1;
        }
      }
      cASSERT0(mc, is_leaf(mc->pg[mc->top]) || page_type(mc->pg[mc->top]) == pagetype);
      cASSERT0(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1]));
      return page_retire(mc, mp);
    }
    DEBUG("root page %" PRIaPGNO " doesn't need rebalancing (flags 0x%x)", mp->pgno, mp->flags);
    return MDBX_SUCCESS;
  }

  /* The parent (branch page) must have at least 2 pointers,
   * otherwise the tree is invalid. */
  const size_t pre_top = mc->top - 1;
  cASSERT0(mc, is_branch(mc->pg[pre_top]));
  cASSERT0(mc, !is_subpage(mc->pg[0]));
  cASSERT0(mc, page_numkeys(mc->pg[pre_top]) > 1);

  /* Leaf page fill factor is below the threshold.
   * Try to move keys from left or right neighbor, or
   * merge with a neighbor page. */

  /* Find neighbors. */
  cursor_couple_t couple;
  MDBX_cursor *const mn = cursor_clone_slightly(mc, &couple);

  page_t *left = nullptr, *right = nullptr;
  if (mn->ki[pre_top] > 0) {
    rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] - 1)), &left, mc->pg[mc->top]->txnid);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    cASSERT0(mc, page_type(left) == page_type(mc->pg[mc->top]));
  }
  if (mn->ki[pre_top] + (size_t)1 < page_numkeys(mn->pg[pre_top])) {
    rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] + (size_t)1)), &right,
                  mc->pg[mc->top]->txnid);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
    cASSERT0(mc, page_type(right) == page_type(mc->pg[mc->top]));
  }
  cASSERT0(mc, left || right);

  const size_t ki_top = mc->ki[mc->top];
  const size_t ki_pre_top = mn->ki[pre_top];

  const size_t left_room = left ? page_room(left) : 0;
  const size_t right_room = right ? page_room(right) : 0;
  const size_t left_nkeys = left ? page_numkeys(left) : 0;
  const size_t right_nkeys = right ? page_numkeys(right) : 0;

  /* Нужно выбрать между правой и левой страницами для слияния текущей или перемещения узла в текущую.
   * Таким образом, нужно выбрать один из четырёх вариантов согласно критериям.
   *
   * Если включен minimize_waf, то стараемся не вовлекать чистые страницы,
   * пренебрегая идеальностью баланса ради уменьшения WAF.
   *
   * При этом отдельные варианты могут быть не доступны, либо "не сработать" из-за того что:
   *  - в какой-то branch-странице не хватит места из-за распространения/обновления первых ключей,
   *    которые хранятся в родительских страницах;
   *  - при включенном minimize_waf распространение/обновление первых ключей
   *    потребуется разделение какой-либо страницы, что увеличит WAF и поэтому обесценивает дальнейшее
   *    следование minimize_waf. */

  if (unlikely(numkeys == 0)) {
    if (left) {
      mn->pg[mn->top] = left;
      mn->ki[mn->top - 1] = (indx_t)(ki_pre_top - 1);
      mn->ki[mn->top] = (indx_t)(left_nkeys - 1);
      mc->ki[mc->top] = 0;
      const size_t new_ki = ki_top + left_nkeys;
      mn->ki[mn->top] += mc->ki[mn->top] + 1;
      couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
      mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
      rc = page_merge(mc, mn);
      mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
      if (likely(rc != MDBX_RESULT_TRUE)) {
        cursor_cpstk(mn, mc);
        mc->ki[mc->top] = (indx_t)new_ki;
        cASSERT0(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys);
        return rc;
      }
    }
    cASSERT0(mc, right_nkeys >= minkeys);
    mn->pg[mn->top] = right;
    mn->ki[mn->top - 1] = (indx_t)(ki_pre_top + 1);
    mn->ki[mn->top] = 0;
    mc->ki[mc->top] = (indx_t)numkeys;
    couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
    mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
    rc = page_merge(mn, mc);
    mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
    if (likely(rc != MDBX_RESULT_TRUE)) {
      mc->ki[mc->top] = (indx_t)ki_top;
      cASSERT0(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys);
      return rc;
    }

  bailout:
    ERROR("Unable to merge/rebalance %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)",
          is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10,
          page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room);
    return MDBX_PROBLEM;
  }

  bool involve = !(left && right);
retry:
  cASSERT0(mc, mc->top > 0);
  const bool consider_left = left && (involve || is_modifable(mc->txn, left));
  const bool consider_right = right && (involve || is_modifable(mc->txn, right));
  if (consider_left && left_room > room_threshold && left_room >= right_room) {
    /* try merge with left */
    cASSERT0(mc, left_nkeys >= minkeys);
    mn->pg[mn->top] = left;
    mn->ki[mn->top - 1] = (indx_t)(ki_pre_top - 1);
    mn->ki[mn->top] = (indx_t)(left_nkeys - 1);
    mc->ki[mc->top] = 0;
    const size_t new_ki = ki_top + left_nkeys;
    mn->ki[mn->top] += mc->ki[mn->top] + 1;
    couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
    mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
    rc = page_merge(mc, mn);
    mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
    if (likely(rc != MDBX_RESULT_TRUE)) {
      cursor_cpstk(mn, mc);
      mc->ki[mc->top] = (indx_t)new_ki;
      cASSERT0(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys);
      return rc;
    }
  }
  if (consider_right && right_room > room_threshold) {
    /* try merge with right */
    cASSERT0(mc, right_nkeys >= minkeys);
    mn->pg[mn->top] = right;
    mn->ki[mn->top - 1] = (indx_t)(ki_pre_top + 1);
    mn->ki[mn->top] = 0;
    mc->ki[mc->top] = (indx_t)numkeys;
    couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
    mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
    rc = page_merge(mn, mc);
    mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
    if (likely(rc != MDBX_RESULT_TRUE)) {
      mc->ki[mc->top] = (indx_t)ki_top;
      cASSERT0(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys);
      return rc;
    }
  }

  if (consider_left && left_nkeys > minkeys && (right_nkeys <= left_nkeys || right_room >= left_room)) {
    /* try move from left */
    mn->pg[mn->top] = left;
    mn->ki[mn->top - 1] = (indx_t)(ki_pre_top - 1);
    mn->ki[mn->top] = (indx_t)(left_nkeys - 1);
    mc->ki[mc->top] = 0;
    couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
    mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
    rc = node_move(mn, mc, true);
    mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
    if (likely(rc != MDBX_RESULT_TRUE)) {
      mc->ki[mc->top] = (indx_t)(ki_top + 1);
      cASSERT0(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys);
      return rc;
    }
  }
  if (consider_right && right_nkeys > minkeys) {
    /* try move from right */
    mn->pg[mn->top] = right;
    mn->ki[mn->top - 1] = (indx_t)(ki_pre_top + 1);
    mn->ki[mn->top] = 0;
    mc->ki[mc->top] = (indx_t)numkeys;
    couple.outer.next = mn->txn->cursors[cursor_dbi(mn)];
    mn->txn->cursors[cursor_dbi(mn)] = &couple.outer;
    rc = node_move(mn, mc, false);
    mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next;
    if (likely(rc != MDBX_RESULT_TRUE)) {
      mc->ki[mc->top] = (indx_t)ki_top;
      cASSERT0(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys);
      return rc;
    }
  }

  if (numkeys >= minkeys) {
    mc->ki[mc->top] = (indx_t)ki_top;
    if (CHECKS2_ENABLED())
      ENSURE_OBJ(mc, cursor_validate_updating(mc) == MDBX_SUCCESS);
    return MDBX_SUCCESS;
  }

  if (minimize_waf && room_threshold > 0) {
    /* Если включен minimize_waf, то переходим к попыткам слияния с сильно
     * заполненными страницами до вовлечения чистых страниц (не измененных в этой транзакции) */
    room_threshold = 0;
    goto retry;
  }
  if (!involve) {
    /* Теперь допускаем вовлечение чистых страниц (не измененных в этой транзакции),
     * что улучшает баланс в дереве, но увеличивает WAF. */
    involve = true;
    goto retry;
  }
  if (room_threshold > 0) {
    /* Если не нашли подходящей соседней, то допускаем слияние с сильно заполненными страницами */
    room_threshold = 0;
    goto retry;
  }

  goto bailout;
}

int tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key) {
  page_t *mp;
  node_t *node;
  size_t len;
  ptrdiff_t delta, ksize, oksize;
  intptr_t ptr, i, nkeys, indx;
  DKBUF_DEBUG;

  cASSERT0(mc, cursor_is_tracked(mc));
  indx = mc->ki[mc->top];
  mp = mc->pg[mc->top];
  node = page_node(mp, indx);
  ptr = mp->entries[indx];
#if MDBX_DEBUG > 0
  MDBX_val k2;
  k2.iov_base = node_key(node);
  k2.iov_len = node_ks(node);
  DEBUG("update key %zi (offset %zu) [%s] to [%s] on page %" PRIaPGNO, indx, ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key),
        mp->pgno);
#endif /* MDBX_DEBUG */

  /* Sizes must be 2-byte aligned. */
  ksize = EVEN_CEIL(key->iov_len);
  oksize = EVEN_CEIL(node_ks(node));
  delta = ksize - oksize;

  /* Shift node contents if EVEN_CEIL(key length) changed. */
  if (delta) {
    if (delta > (int)page_room(mp)) {
      /* not enough space left, do a delete and split */
      DEBUG("Not enough room, delta = %zd, splitting...", delta);
      pgno_t pgno = node_pgno(node);
      node_del(mc, 0);
      int err = page_split(mc, key, nullptr, pgno, MDBX_SPLIT_REPLACE);
      if (err == MDBX_SUCCESS && CHECKS2_ENABLED())
        ENSURE_OBJ(mc, cursor_validate_updating(mc) == MDBX_SUCCESS);
      return err;
    }

    nkeys = page_numkeys(mp);
    for (i = 0; i < nkeys; i++) {
      if (mp->entries[i] <= ptr) {
        cASSERT0(mc, mp->entries[i] >= delta);
        mp->entries[i] -= (indx_t)delta;
      }
    }

    void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ);
    len = ptr - mp->upper + NODESIZE;
    memmove(ptr_disp(base, -delta), base, len);
    cASSERT0(mc, mp->upper >= delta);
    mp->upper -= (indx_t)delta;

    node = page_node(mp, indx);
  }

  /* But even if no shift was needed, update ksize */
  node_set_ks(node, key->iov_len);

  if (likely(key->iov_len /* to avoid UBSAN traps*/ != 0))
    memcpy(node_key(node), key->iov_base, key->iov_len);
  return MDBX_SUCCESS;
}

/* Search for the lowest key under the current branch page. This just bypasses a numkeys check in the current page
 * before calling tree_search_continue(), because the callers are all in situations where the current page is known
 * to be underfilled. */
__hot __noinline int tree_deepen_lowest(MDBX_cursor *mc) {
  cASSERT0(mc, mc->top >= 0);
  page_t *mp = mc->pg[mc->top];
  cASSERT0(mc, is_branch(mp));

  node_t *node = page_node(mp, 0);
  int err = page_get(mc, node_pgno(node), &mp, mp->txnid);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  mc->ki[mc->top] = 0;
  err = cursor_push(mc, mp, 0);
  if (unlikely(err != MDBX_SUCCESS))
    return err;
  return tree_deepen_edge(mc, Z_FIRST);
}

__hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) {
  int err;
  if (unlikely(mc->txn->flags & MDBX_TXN_BLOCKED)) {
    DEBUG("%s", "transaction has failed, must abort");
    err = MDBX_BAD_TXN;
  bailout:
    be_poor(mc);
    return err;
  }

  const size_t dbi = cursor_dbi(mc);
  if (unlikely(*cursor_dbi_state(mc) & DBI_STALE)) {
    err = tbl_refresh_absent2baddbi(mc->txn, dbi);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;
  }

  const pgno_t root = mc->tree->root;
  if (unlikely(root == P_INVALID)) {
    DEBUG("%s", "tree is empty");
    cASSERT0(mc, is_poor(mc));
    return MDBX_NOTFOUND;
  }

  cASSERT0(mc, root >= NUM_METAS && root < mc->txn->geo.first_unallocated);
  if (mc->top < 0 || mc->pg[0]->pgno != root) {
    err = page_get(mc, root, &mc->pg[0], tbl_root_txnid(mc->txn, cursor_dbi(mc)));
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;
  }

  mc->top = 0;
  mc->ki[0] = (flags & Z_LAST) ? page_numkeys(mc->pg[0]) - 1 : 0;
  DEBUG("db %d root page %" PRIaPGNO " has flags 0x%X", cursor_dbi_dbg(mc), root, mc->pg[0]->flags);

  if (flags & Z_MODIFY) {
    err = page_touch(mc);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;
  }

  if (flags & Z_ROOTONLY)
    return MDBX_SUCCESS;

  if (flags & (Z_FIRST | Z_LAST))
    return tree_deepen_edge(mc, flags);

  DKBUF_DEBUG;
  page_t *mp = mc->pg[mc->top];
  while (is_branch(mp)) {
    DEBUG("branch page %" PRIaPGNO " has %zu keys", mp->pgno, page_numkeys(mp));
    cASSERT0(mc, page_numkeys(mp) > 1);
    TRACE("found index 0 to page %" PRIaPGNO, node_pgno(page_node(mp, 0)));

    const intptr_t ki = tree_search_branch(mc, key);
    TRACE("following index %zu for key [%s]", ki, DKEY_DEBUG(key));

    mc->ki[mc->top] = (indx_t)ki;
    err = page_get(mc, node_pgno(page_node(mp, ki)), &mp, mp->txnid);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;

    err = cursor_push(mc, mp, 0);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;

    if (unlikely(flags & Z_MODIFY)) {
      err = page_touch(mc);
      if (unlikely(err != MDBX_SUCCESS))
        goto bailout;
      mp = mc->pg[mc->top];
    }
  }

  if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
    ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
    err = MDBX_CORRUPTED;
    goto bailout;
  }

  DEBUG("found leaf page %" PRIaPGNO " for key [%s]", mp->pgno, DKEY_DEBUG(key));
  return MDBX_SUCCESS;
}

__hot __noinline int tree_deepen_edge(MDBX_cursor *mc, int flags) {
  cASSERT0(mc, !is_poor(mc));
  int err;
  page_t *mp = mc->pg[mc->top];
  while (is_branch(mp)) {
    DEBUG("branch page %" PRIaPGNO " has %zu keys", mp->pgno, page_numkeys(mp));
    cASSERT0(mc, page_numkeys(mp) > 1);

    err = page_get(mc, node_pgno(page_node(mp, mc->ki[mc->top])), &mp, mp->txnid);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;

    err = cursor_push(mc, mp, (flags & Z_FIRST) ? 0 : page_numkeys(mp) - 1);
    if (unlikely(err != MDBX_SUCCESS))
      goto bailout;

    if (unlikely(flags & Z_MODIFY)) {
      err = page_touch(mc);
      if (unlikely(err != MDBX_SUCCESS))
        goto bailout;
      mp = mc->pg[mc->top];
    }
  }

  if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
    ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
    err = MDBX_CORRUPTED;
  bailout:
    be_poor(mc);
    return err;
  }

  DEBUG("seek leaf page %" PRIaPGNO " for @%s", mp->pgno, (flags & Z_FIRST) ? "first" : "last");
  return MDBX_SUCCESS;
}

/* ---------------------------------------------------------------------------------------------------- */

#if defined(__GNUC__) && !(defined(__e2k__) || defined(__elbrus__))
#define CLEAR_VALUE_PROPAGATION(VAR) __asm__ __volatile__("" : "+r"(cmp))
#else
#define CLEAR_VALUE_PROPAGATION(VAR)                                                                                   \
  do {                                                                                                                 \
  } while (0)
#endif

#define BINARY_BRANCHLESS_SEARCH_CYCLE_BEGIN(IT, CMP, LOWER, SIZE)                                                     \
  intptr_t adjust = SIZE - 1, half = adjust >> 1;                                                                      \
  adjust &= 1;                                                                                                         \
  IT = LOWER + half

#define BINARY_BRANCHLESS_SEARCH_CYCLE_END(IT, CMP, LOWER, SIZE)                                                       \
  CMP >>= CHAR_BIT * sizeof(CMP) - 1;                                                                                  \
  CLEAR_VALUE_PROPAGATION(CMP);                                                                                        \
  LOWER += (half + 1) & CMP;                                                                                           \
  SIZE = half + (CMP & adjust)

static int null_comparator(const MDBX_val *a, const MDBX_val *b) {
  (void)a;
  (void)b;
  panic("must not be called");
  return 0;
}

#define SEARCH_FOLIAGE(NAME, BRANCH_COMPARATOR, LEAF_COMPARATOR, BOOL_DIFFERENT_COMPARATORS, BOOL_DUPFIXED)            \
  MDBX_NOTHROW_PURE_FUNCTION __hot static sfr_t search_foliage_##NAME(MDBX_cursor *mc, const MDBX_val *key) {          \
    page_t *mp = mc->pg[mc->top];                                                                                      \
    const intptr_t nkeys = page_numkeys(mp);                                                                           \
    DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys, is_leaf(mp) ? "leaf" : "branch",                        \
          is_subpage(mp) ? "sub-" : "", mp->pgno);                                                                     \
                                                                                                                       \
    sfr_t ret;                                                                                                         \
    ret.exact = false;                                                                                                 \
    STATIC_ASSERT(P_BRANCH == 1);                                                                                      \
    intptr_t lo = mp->flags & P_BRANCH;                                                                                \
    intptr_t hi = nkeys - 1;                                                                                           \
    if (unlikely(hi < lo)) {                                                                                           \
      mc->ki[mc->top] = 0;                                                                                             \
      ret.node = nullptr;                                                                                              \
      return ret;                                                                                                      \
    }                                                                                                                  \
                                                                                                                       \
    intptr_t scope = nkeys - lo, cmp, it;                                                                              \
    node_t *node = (node_t *)(intptr_t)-1;                                                                             \
    MDBX_val node_key;                                                                                                 \
    if ((BOOL_DIFFERENT_COMPARATORS | BOOL_DUPFIXED) && lo == 0) {                                                     \
      ASSERT(is_leaf(mp));                                                                                             \
      if (BOOL_DUPFIXED) {                                                                                             \
        cASSERT0(mc, is_dupfix_leaf(mp));                                                                              \
        cASSERT0(mc, mp->dupfix_ksize == mc->tree->dupfix_size);                                                       \
        node_key.iov_len = mp->dupfix_ksize;                                                                           \
        TRACE(">> %s lo %zu, size %zu, nkeys %zu", "leaf-dupfix", lo, scope, nkeys);                                   \
        do {                                                                                                           \
          MDBX_CURSOR_STC_INC(mc);                                                                                     \
          BINARY_BRANCHLESS_SEARCH_CYCLE_BEGIN(it, cmp, lo, scope);                                                    \
          node_key.iov_base = page_dupfix_ptr(mp, it, node_key.iov_len);                                               \
          cASSERT0(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(node_key.iov_base, node_key.iov_len));               \
          cmp = LEAF_COMPARATOR(&node_key, key);                                                                       \
          TRACE("== i %zu, cmp %zi", it, cmp);                                                                         \
          if (unlikely(cmp == 0))                                                                                      \
            goto found;                                                                                                \
          BINARY_BRANCHLESS_SEARCH_CYCLE_END(it, cmp, lo, scope);                                                      \
          TRACE("== lo %zi, size %zi", lo, scope);                                                                     \
        } while (scope > 0);                                                                                           \
                                                                                                                       \
        it += cmp & 1;                                                                                                 \
        /* store the key index */                                                                                      \
        mc->ki[mc->top] = (indx_t)it;                                                                                  \
        ret.node =                                                                                                     \
            (it < nkeys) ? /* fake for DUPFIX */ node : /* There is no entry larger or equal to the key. */ nullptr;   \
        TRACE("<< lo %zu, size %zu, nkeys %zu, i %zu, %c", lo, scope, nkeys, it, 'N');                                 \
        return ret;                                                                                                    \
      } else {                                                                                                         \
        cASSERT0(mc, !is_dupfix_leaf(mp));                                                                             \
        TRACE(">> %s lo %zu, size %zu, nkeys %zu", "leaf", lo, scope, nkeys);                                          \
        do {                                                                                                           \
          MDBX_CURSOR_STC_INC(mc);                                                                                     \
          BINARY_BRANCHLESS_SEARCH_CYCLE_BEGIN(it, cmp, lo, scope);                                                    \
          node = page_node(mp, it);                                                                                    \
          node_key = get_key(node);                                                                                    \
          cASSERT0(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(node_key.iov_base, node_key.iov_len));               \
          cmp = LEAF_COMPARATOR(&node_key, key);                                                                       \
          TRACE("== i %zu, cmp %zi", it, cmp);                                                                         \
          if (unlikely(cmp == 0))                                                                                      \
            goto found;                                                                                                \
          BINARY_BRANCHLESS_SEARCH_CYCLE_END(it, cmp, lo, scope);                                                      \
          TRACE("== lo %zi, size %zi", lo, scope);                                                                     \
        } while (scope > 0);                                                                                           \
                                                                                                                       \
        it += cmp & 1;                                                                                                 \
        /* store the key index */                                                                                      \
        mc->ki[mc->top] = (indx_t)it;                                                                                  \
        ret.node = (it < nkeys) ? page_node(mp, it) : /* There is no entry larger or equal to the key. */ nullptr;     \
        TRACE("<< lo %zu, size %zu, nkeys %zu, i %zu, %c", lo, scope, nkeys, it, 'N');                                 \
        return ret;                                                                                                    \
      }                                                                                                                \
    }                                                                                                                  \
                                                                                                                       \
    TRACE(">> %s lo %zu, size %zu, nkeys %zu", "branch", lo, scope, nkeys);                                            \
    ASSERT(!is_dupfix_leaf(mp));                                                                                       \
    ASSERT(is_branch(mp) || !(BOOL_DIFFERENT_COMPARATORS | BOOL_DUPFIXED));                                            \
    do {                                                                                                               \
      MDBX_CURSOR_STC_INC(mc);                                                                                         \
      BINARY_BRANCHLESS_SEARCH_CYCLE_BEGIN(it, cmp, lo, scope);                                                        \
      node = page_node(mp, it);                                                                                        \
      node_key = get_key(node);                                                                                        \
      cASSERT0(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(node_key.iov_base, node_key.iov_len));                   \
      cmp = BRANCH_COMPARATOR(&node_key, key);                                                                         \
      TRACE("== i %zu, cmp %zi", it, cmp);                                                                             \
      if (unlikely(cmp == 0))                                                                                          \
        goto found;                                                                                                    \
      BINARY_BRANCHLESS_SEARCH_CYCLE_END(it, cmp, lo, scope);                                                          \
      TRACE("== lo %zi, size %zi", lo, scope);                                                                         \
    } while (scope > 0);                                                                                               \
                                                                                                                       \
    it += cmp & 1;                                                                                                     \
    /* store the key index */                                                                                          \
    mc->ki[mc->top] = (indx_t)it;                                                                                      \
    ret.node = (it < nkeys) ? page_node(mp, it) : /* There is no entry larger or equal to the key. */ nullptr;         \
    TRACE("<< lo %zu, size %zu, nkeys %zu, i %zu, %c", lo, scope, nkeys, it, 'N');                                     \
    return ret;                                                                                                        \
                                                                                                                       \
  found:                                                                                                               \
    TRACE("<< lo %zu, size %zu, nkeys %zu, i %zu, %c", lo, scope, nkeys, it, 'Y');                                     \
    mc->ki[mc->top] = (indx_t)it;                                                                                      \
    ret.node = node;                                                                                                   \
    ret.exact = true;                                                                                                  \
    return ret;                                                                                                        \
  }

SEARCH_FOLIAGE(lexical_usual, cmp_lexical, null_comparator, false, false)
SEARCH_FOLIAGE(reverse_usual, cmp_reverse, null_comparator, false, false)
SEARCH_FOLIAGE(lenfast_usual, cmp_lenfast, null_comparator, false, false)
SEARCH_FOLIAGE(custom_usual, mc->clc->k.cmp, null_comparator, false, false)

#if defined(cmp_uint_align4)
SEARCH_FOLIAGE(ordinal_usual, cmp_uint_align4, null_comparator, false, false)
#else
SEARCH_FOLIAGE(ordinal_usual, cmp_uint_align4, cmp_uint_unaligned, true, false)
#endif

#if defined(cmp_uint32_align4_unchecked)
SEARCH_FOLIAGE(uint32_usual, cmp_uint32_align4_unchecked, null_comparator, false, false)
#else
SEARCH_FOLIAGE(uint32_usual, cmp_uint32_align4_unchecked, cmp_uint32_unaligned_unchecked, true, false)
#endif

#if defined(cmp_uint64_align4_unchecked)
SEARCH_FOLIAGE(uint64_usual, cmp_uint64_align4_unchecked, null_comparator, false, false)
#else
SEARCH_FOLIAGE(uint64_usual, cmp_uint64_align4_unchecked, cmp_uint64_unaligned_unchecked, true, false)
#endif

SEARCH_FOLIAGE(lexical_dupfix, cmp_lexical, cmp_lexical, false, true)
SEARCH_FOLIAGE(reverse_dupfix, cmp_reverse, cmp_reverse, false, true)
SEARCH_FOLIAGE(lenfast_dupfix, cmp_lenfast, cmp_lenfast, false, true)
SEARCH_FOLIAGE(custom_dupfix, mc->clc->k.cmp, mc->clc->k.cmp, false, true)

SEARCH_FOLIAGE(ordinal_dupfix, cmp_uint_align4, cmp_uint_unaligned, true, true)
SEARCH_FOLIAGE(uint32_dupfix, cmp_uint32_align4_unchecked, cmp_uint32_unaligned_unchecked, true, true)
SEARCH_FOLIAGE(uint64_dupfix, cmp_uint64_align4_unchecked, cmp_uint64_unaligned_unchecked, true, true)

#if MDBX_DEBUG_SEARCH_DISPATCHING

MDBX_MAYBE_UNUSED __cold static sfr_t old_node_search(MDBX_cursor *mc, const MDBX_val *key) {
  page_t *mp = mc->pg[mc->top];
  const intptr_t nkeys = page_numkeys(mp);
  DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys, is_leaf(mp) ? "leaf" : "branch",
        is_subpage(mp) ? "sub-" : "", mp->pgno);

  sfr_t ret;
  ret.exact = false;
  STATIC_ASSERT(P_BRANCH == 1);
  intptr_t lo = mp->flags & P_BRANCH;
  intptr_t hi = nkeys - 1;
  if (unlikely(hi < lo)) {
    mc->ki[mc->top] = 0;
    ret.node = nullptr;
    return ret;
  }

  intptr_t i;
  MDBX_cmp_func comparator = mc->clc->k.cmp;
  MDBX_val nodekey;
  if (unlikely(is_dupfix_leaf(mp))) {
    cASSERT0(mc, mp->dupfix_ksize == mc->tree->dupfix_size);
    nodekey.iov_len = mp->dupfix_ksize;
    TRACE(">> lo %zi, hi %zi, size %zi, nkeys %zu", lo, hi, hi - lo + 1, nkeys);
    do {
      MDBX_CURSOR_STC_INC(mc);
      i = (lo + hi) >> 1;
      nodekey.iov_base = page_dupfix_ptr(mp, i, nodekey.iov_len);
      cASSERT0(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len));
      int cmp = comparator(key, &nodekey);
      TRACE("== i %zu, cmp %i", i, cmp);
      if (cmp > 0)
        lo = ++i;
      else if (cmp < 0)
        hi = i - 1;
      else {
        ret.exact = true;
        break;
      }
      TRACE("== lo %zi, hi %zi, size %zi", lo, hi, hi - lo + 1);
    } while (likely(lo <= hi));

    TRACE("<< lo %zi, hi %zi, size %zi, nkeys %zu, i %zu, %c", lo, hi, hi - lo + 1, nkeys, i, ret.exact ? 'Y' : 'N');

    /* store the key index */
    mc->ki[mc->top] = (indx_t)i;
    ret.node = (i < nkeys) ? /* fake for DUPFIX */ (node_t *)(intptr_t)-1
                           : /* There is no entry larger or equal to the key. */ nullptr;
    return ret;
  }

  if (MDBX_UNALIGNED_OK < 4 && is_branch(mp) && comparator == cmp_uint_align2)
    /* Branch pages have no data, so if using integer keys,
     * alignment is guaranteed. Use faster cmp_uint_align4(). */
    comparator = cmp_uint_align4;

  node_t *node;
  TRACE(">> lo %zi, hi %zi, size %zi, nkeys %zu", lo, hi, hi - lo + 1, nkeys);
  do {
    MDBX_CURSOR_STC_INC(mc);
    i = (lo + hi) >> 1;
    node = page_node(mp, i);
    nodekey.iov_len = node_ks(node);
    nodekey.iov_base = node_key(node);
    cASSERT0(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len));
    int cmp = comparator(key, &nodekey);
    TRACE("== i %zu, cmp %i", i, cmp);
    if (cmp > 0)
      lo = ++i;
    else if (cmp < 0)
      hi = i - 1;
    else {
      ret.exact = true;
      break;
    }
    TRACE("== lo %zi, hi %zi, size %zi", lo, hi, hi - lo + 1);
  } while (likely(lo <= hi));

  TRACE("<< lo %zi, hi %zi, size %zi, nkeys %zu, i %zu, %c", lo, hi, hi - lo + 1, nkeys, i, ret.exact ? 'Y' : 'N');

  /* store the key index */
  mc->ki[mc->top] = (indx_t)i;
  ret.node = (i < nkeys) ? page_node(mp, i) : /* There is no entry larger or equal to the key. */ nullptr;
  return ret;
}

MDBX_MAYBE_UNUSED __cold static size_t old_branch_search(MDBX_cursor *mc, const MDBX_val *key) {
  sfr_t nsr = old_node_search(mc, key);
  size_t indx = page_numkeys(mc->pg[mc->top]) - 1;
  if (likely(nsr.node))
    indx = mc->ki[mc->top] + (intptr_t)nsr.exact - 1;
  return indx;
}

#endif /* MDBX_DEBUG_SEARCH_DISPATCHING */

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot static MDBX_search_foliage
cursor_to_search_foliage(const MDBX_cursor *mc) {
  MDBX_cmp_func comparator = mc->clc->k.cmp;
  ASSERT(comparator != nullptr);

  if ((mc->tree->flags & MDBX_DUPFIXED) && is_inner(mc)) {

    if (comparator == cmp_lexical)
      return search_foliage_lexical_dupfix;
    if (comparator == cmp_reverse)
      return search_foliage_reverse_dupfix;
    if (comparator == cmp_lenfast)
      return search_foliage_lenfast_dupfix;

    if (mc->tree->flags & MDBX_INTEGERKEY) {
      const size_t keylen = mc->tree->dupfix_size;
      cASSERT0(mc, keylen >= mc->clc->k.lmin && keylen <= mc->clc->k.lmax && (keylen == 4 || keylen == 8));
      if (/* paranoia */ keylen >= mc->clc->k.lmin && keylen <= mc->clc->k.lmax && (keylen == 4 || keylen == 8)) {
        mc->clc->k.lmin = keylen;
        mc->clc->k.lmax = keylen;
      }
      size_t ordinal = 0;
#ifndef cmp_uint_align2
      if (comparator == cmp_uint_align2)
        ordinal = keylen;
#endif /* cmp_uint_align2 */
#ifndef cmp_uint_align4
      if (comparator == cmp_uint_align4)
        ordinal = keylen;
#endif /* cmp_uint_align4 */
      if (comparator == cmp_uint_unaligned)
        ordinal = keylen;
      if (ordinal) {
        if ((mc->txn->env->flags & MDBX_VALIDATION) == 0 && ordinal == mc->clc->k.lmin && ordinal == mc->clc->k.lmax) {
          if (ordinal == 4)
            return search_foliage_uint32_dupfix;
          if (ordinal == 8)
            return search_foliage_uint64_dupfix;
        }
        return search_foliage_ordinal_dupfix;
      }
    }

    return search_foliage_custom_dupfix;
  }

  if (comparator == cmp_lexical)
    return search_foliage_lexical_usual;
  if (comparator == cmp_reverse)
    return search_foliage_reverse_usual;
  if (comparator == cmp_lenfast)
    return search_foliage_lenfast_usual;

  if (mc->tree->flags & MDBX_INTEGERKEY) {
    cASSERT0(mc, mc->top >= 0);
    page_t *mp = mc->pg[mc->top];
    cASSERT0(mc, !is_dupfix_leaf(mp) && page_numkeys(mp) > 0);
    const size_t keylen = node_ks(page_node(mp, 0));
    cASSERT0(mc, keylen >= mc->clc->k.lmin && keylen <= mc->clc->k.lmax && (keylen == 4 || keylen == 8));
    if (/* paranoia */ keylen >= mc->clc->k.lmin && keylen <= mc->clc->k.lmax && (keylen == 4 || keylen == 8)) {
      mc->clc->k.lmin = keylen;
      mc->clc->k.lmax = keylen;
    }
    size_t ordinal = 0;
#ifndef cmp_uint_align2
    if (comparator == cmp_uint_align2)
      ordinal = keylen;
#endif /* cmp_uint_align2 */
#ifndef cmp_uint_align4
    if (comparator == cmp_uint_align4)
      ordinal = keylen;
#endif /* cmp_uint_align4 */
    if (comparator == cmp_uint_unaligned)
      ordinal = keylen;
    if (ordinal) {
      if ((mc->txn->env->flags & MDBX_VALIDATION) == 0 && ordinal == mc->clc->k.lmin && ordinal == mc->clc->k.lmax) {
        if (ordinal == 4)
          return search_foliage_uint32_usual;
        if (ordinal == 8)
          return search_foliage_uint64_usual;
      }
      return search_foliage_ordinal_usual;
    }
  }

  return search_foliage_custom_usual;
}

__hot sfr_t tree_search_foliage_configure(MDBX_cursor *mc, const MDBX_val *key) {
  MDBX_search_foliage search_foliage = cursor_to_search_foliage(mc);
#if MDBX_DEBUG_SEARCH_DISPATCHING
  const unsigned snap_1 = MDBX_CURSOR_STC_GET(mc);
  sfr_t old = old_node_search(mc, key);
  int old_i = mc->ki[mc->top];
  const unsigned snap_2 = MDBX_CURSOR_STC_GET(mc);
  const unsigned old_steps = snap_2 - snap_1;

  sfr_t new = search_foliage(mc, key);
  int new_i = mc->ki[mc->top];
  const unsigned snap_3 = MDBX_CURSOR_STC_GET(mc);
  const unsigned new_steps = snap_3 - snap_2;

  if (new_i != old_i || new.exact != old.exact || new.node != old.node || new_steps > old_steps) {
    globals.loglevel = MDBX_LOG_TRACE;
    WARNING("\nfoliage-search-issue: new %i, %c, %p, steps %u | old %i, %c, %p, steps %u | retry to debug...", new_i,
            new.exact ? 'Y' : 'N', (void *)new.node, new_steps, old_i, old.exact ? 'Y' : 'N', (void *)old.node,
            old_steps);
    old_node_search((MDBX_cursor *)mc, key);
    search_foliage(mc, key);
    panic_fmt(mc, "new %i, %c, %p, steps %u | old %i, %c, %p, steps %u", new_i, new.exact ? 'Y' : 'N', (void *)new.node,
              new_steps, old_i, old.exact ? 'Y' : 'N', (void *)old.node, old_steps);
  }
  return new;
#else
  mc->clc->k.search_foliage = search_foliage;
  return search_foliage(mc, key);
#endif /* MDBX_DEBUG_SEARCH_DISPATCHING */
}

/* ---------------------------------------------------------------------------------------------------- */

#define SEARCH_BRANCH(NAME, COMPARATOR)                                                                                \
  MDBX_NOTHROW_PURE_FUNCTION __hot static size_t search_branch_##NAME(const MDBX_cursor *mc, const MDBX_val *key) {    \
    page_t *mp = mc->pg[mc->top];                                                                                      \
    ASSERT(is_branch(mp));                                                                                             \
    const size_t nkeys = page_numkeys(mp);                                                                             \
    cASSERT0(mc, nkeys >= 2);                                                                                          \
    TRACE("searching %zu keys in branch-page %" PRIaPGNO, nkeys, mp->pgno);                                            \
    intptr_t lo = 1, scope = nkeys - lo, it, cmp;                                                                      \
    TRACE(">> lo %zu, size %zu, nkeys %zu", lo, scope, nkeys);                                                         \
    if (likely(nkeys > 1))                                                                                             \
      do {                                                                                                             \
        MDBX_CURSOR_STC_INC(mc);                                                                                       \
        BINARY_BRANCHLESS_SEARCH_CYCLE_BEGIN(it, cmp, lo, scope);                                                      \
        MDBX_val node_key = get_key(page_node(mp, it));                                                                \
        cASSERT0(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(node_key.iov_base, node_key.iov_len));                 \
        cmp = COMPARATOR(&node_key, key);                                                                              \
        TRACE("== i %zu, cmp %zi", it, cmp);                                                                           \
        if (unlikely(cmp == 0)) {                                                                                      \
          TRACE("<< lo %zu, size %zu, nkeys %zu, i %zu, %c", lo, scope, nkeys, it, 'Y');                               \
          return it;                                                                                                   \
        }                                                                                                              \
        BINARY_BRANCHLESS_SEARCH_CYCLE_END(it, cmp, lo, scope);                                                        \
        TRACE("== lo %zi, size %zi", lo, scope);                                                                       \
      } while (likely(scope > 0));                                                                                     \
    it = lo - 1;                                                                                                       \
    TRACE("<< lo %zu, size %zu, nkeys %zu, i %zu, %c", lo, scope, nkeys, it, 'N');                                     \
    return it;                                                                                                         \
  }

/* Branch pages have no data, so if using integer keys, alignment is guaranteed. Use faster cmp_uint_align4(). */
SEARCH_BRANCH(ordinal, cmp_uint_align4)
SEARCH_BRANCH(uint32, cmp_uint32_align4_unchecked)
SEARCH_BRANCH(uint64, cmp_uint64_align4_unchecked)
SEARCH_BRANCH(lexical, cmp_lexical)
SEARCH_BRANCH(reverse, cmp_reverse)
SEARCH_BRANCH(lenfast, cmp_lenfast)
SEARCH_BRANCH(custom, mc->clc->k.cmp)

MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION __hot static MDBX_search_branch
cursor_to_search_branch(const MDBX_cursor *mc) {
  MDBX_cmp_func comparator = mc->clc->k.cmp;
  ASSERT(comparator != nullptr);

  if (comparator == cmp_lexical)
    return search_branch_lexical;
  if (comparator == cmp_reverse)
    return search_branch_reverse;
  if (comparator == cmp_lenfast)
    return search_branch_lenfast;

  if (mc->tree->flags & MDBX_INTEGERKEY) {
    cASSERT0(mc, mc->top >= 0);
    page_t *mp = mc->pg[mc->top];
    cASSERT0(mc, page_numkeys(mp) > 0);
    STATIC_ASSERT(P_BRANCH == 1);
    const size_t keylen = is_dupfix_leaf(mp) ? mp->dupfix_ksize : node_ks(page_node(mp, mp->flags & P_BRANCH));
    cASSERT0(mc, keylen >= mc->clc->k.lmin && keylen <= mc->clc->k.lmax && (keylen == 4 || keylen == 8));
    if (/* paranoia */ keylen >= mc->clc->k.lmin && keylen <= mc->clc->k.lmax && (keylen == 4 || keylen == 8)) {
      mc->clc->k.lmin = keylen;
      mc->clc->k.lmax = keylen;
    }
    size_t ordinal = 0;
#ifndef cmp_uint_align2
    if (comparator == cmp_uint_align2)
      ordinal = keylen;
#endif /* cmp_uint_align2 */
#ifndef cmp_uint_align4
    if (comparator == cmp_uint_align4)
      ordinal = keylen;
#endif /* cmp_uint_align4 */
    if (comparator == cmp_uint_unaligned)
      ordinal = keylen;
    if (ordinal) {
      if ((mc->txn->env->flags & MDBX_VALIDATION) == 0 && ordinal == mc->clc->k.lmin && ordinal == mc->clc->k.lmax) {
        if (ordinal == 4)
          return search_branch_uint32;
        if (ordinal == 8)
          return search_branch_uint64;
      }
      return search_branch_ordinal;
    }
  }

  return search_branch_custom;
}

size_t tree_search_branch_configure(const MDBX_cursor *mc, const MDBX_val *key) {
  MDBX_search_branch search_branch = cursor_to_search_branch(mc);
#if MDBX_DEBUG_SEARCH_DISPATCHING
  const unsigned snap_1 = MDBX_CURSOR_STC_GET(mc);
  size_t old_i = old_branch_search((MDBX_cursor *)mc, key);
  const unsigned snap_2 = MDBX_CURSOR_STC_GET(mc);
  const unsigned old_steps = snap_2 - snap_1;

  size_t new_i = search_branch(mc, key);
  const unsigned snap_3 = MDBX_CURSOR_STC_GET(mc);
  const unsigned new_steps = snap_3 - snap_2;

  if (new_i != old_i || new_steps > old_steps) {
    globals.loglevel = MDBX_LOG_TRACE;
    WARNING("\nbranch-search-issue: new %zi, steps %u | old %zi, steps %u | retry to debug...", new_i, new_steps, old_i,
            old_steps);
    old_branch_search((MDBX_cursor *)mc, key);
    search_branch(mc, key);
    panic_fmt(mc, "new %zi, steps %u | old %zi, steps %u", new_i, new_steps, old_i, old_steps);
  }
  return new_i;
#else
  mc->clc->k.search_branch = search_branch;
  return search_branch(mc, key);
#endif /* MDBX_DEBUG_SEARCH_DISPATCHING */
}

#undef CLEAR_VALUE_PROPAGATION
#undef BINARY_BRANCHLESS_SEARCH_CYCLE_BEGIN
#undef BINARY_BRANCHLESS_SEARCH_CYCLE_END
#undef SEARCH_BRANCH
#undef SEARCH_FOLIAGE

static inline size_t txl_size2bytes(const size_t size) {
  ASSERT(size > 0 && size <= txl_max * 2);
  size_t bytes =
      ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * (size + 2), txl_granulate * sizeof(txnid_t)) -
      MDBX_ASSUME_MALLOC_OVERHEAD;
  return bytes;
}

static inline size_t txl_bytes2size(const size_t bytes) {
  size_t size = bytes / sizeof(txnid_t);
  ASSERT(size > 2 && size <= txl_max * 2);
  return size - 2;
}

txl_t txl_alloc(void) {
  size_t bytes = txl_size2bytes(txl_initial);
  txl_t txl = osal_malloc(bytes);
  if (likely(txl)) {
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(txl);
#endif /* osal_malloc_usable_size */
    txl[0] = txl_bytes2size(bytes);
    ASSERT(txl[0] >= txl_initial);
    txl += 1;
    *txl = 0;
  }
  return txl;
}

void txl_free(txl_t txl) {
  if (likely(txl))
    osal_free(txl - 1);
}

static int txl_reserve(txl_t __restrict *__restrict ptxl, const size_t wanna) {
  const size_t allocated = txl_alloclen(*ptxl);
  ASSERT(txl_size(*ptxl) <= txl_max && txl_alloclen(*ptxl) >= txl_size(*ptxl));
  if (likely(allocated >= wanna))
    return MDBX_SUCCESS;

  if (unlikely(wanna > /* paranoia */ txl_max)) {
    ERROR("TXL too long (%zu > %zu)", wanna, (size_t)txl_max);
    return MDBX_TXN_FULL;
  }

  const size_t size = (wanna + wanna - allocated < txl_max) ? wanna + wanna - allocated : txl_max;
  size_t bytes = txl_size2bytes(size);
  txl_t txl = osal_realloc(*ptxl - 1, bytes);
  if (likely(txl)) {
#ifdef osal_malloc_usable_size
    bytes = osal_malloc_usable_size(txl);
#endif /* osal_malloc_usable_size */
    *txl = txl_bytes2size(bytes);
    ASSERT(*txl >= wanna);
    *ptxl = txl + 1;
    return MDBX_SUCCESS;
  }
  return MDBX_ENOMEM;
}

static inline int __must_check_result txl_need(txl_t __restrict *__restrict ptxl, size_t num) {
  ASSERT(txl_size(*ptxl) <= txl_max && txl_alloclen(*ptxl) >= txl_size(*ptxl));
  ASSERT(num <= PAGELIST_LIMIT);
  const size_t wanna = txl_size(*ptxl) + num;
  return likely(txl_alloclen(*ptxl) >= wanna) ? MDBX_SUCCESS : txl_reserve(ptxl, wanna);
}

static inline void txl_append_prereserved(txl_t __restrict txl, txnid_t id) {
  ASSERT(txl_size(txl) < txl_alloclen(txl));
  size_t end = txl[0] += 1;
  txl[end] = id;
}

#define TXNID_SORT_CMP(first, last) ((first) > (last))
SORT_IMPL(txnid_sort, false, txnid_t, TXNID_SORT_CMP)
void txl_sort(txl_t txl) { txnid_sort(txl + 1, txl + txl_size(txl) + 1); }

int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id) {
  if (unlikely(txl_size(*ptxl) == txl_alloclen(*ptxl))) {
    int rc = txl_need(ptxl, txl_granulate);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }
  txl_append_prereserved(*ptxl, id);
  return MDBX_SUCCESS;
}

__hot bool txl_contain(const txl_t txl, txnid_t id) {
  const size_t len = txl_size(txl);
  for (size_t i = 1; i <= len; ++i)
    if (txl[i] == id)
      return true;
  return false;
}

static int txn_write(MDBX_txn *txn, iov_ctx_t *ctx) {
  cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
  dpl_t *const dl = txn_dpl_sort(txn);
  int rc = MDBX_SUCCESS;
  size_t r, w, total_npages = 0;
  for (w = 0, r = 1; r <= dl->length; ++r) {
    page_t *dp = dl->items[r].ptr;
    if (dp->flags & P_LOOSE) {
      dl->items[++w] = dl->items[r];
      continue;
    }
    unsigned npages = dpl_npages(dl, r);
    total_npages += npages;
    rc = iov_page(txn, ctx, dp, npages);
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }

  if (!iov_empty(ctx)) {
    cASSERT0(txn, rc == MDBX_SUCCESS);
    rc = iov_write(ctx);
  }

  if (likely(rc == MDBX_SUCCESS) && ctx->fd == txn->env->lazy_fd) {
    txn->env->lck->unsynced_pages.weak += total_npages;
    if (!txn->env->lck->eoos_timestamp.weak)
      txn->env->lck->eoos_timestamp.weak = osal_monotime();
  }

  txn->wr.dirtylist->pages_including_loose -= total_npages;
  while (r <= dl->length)
    dl->items[++w] = dl->items[r++];

  dl->sorted = dpl_setlen(dl, w);
  txn->wr.dirtyroom += r - 1 - w;
  cASSERT0(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
                    (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
  cASSERT0(txn, txn->wr.dirtylist->length == txn->wr.loose_count);
  cASSERT0(txn, txn->wr.dirtylist->pages_including_loose == txn->wr.loose_count);
  return rc;
}

__cold MDBX_txn *txn_basal_create(const size_t max_dbi) {
  MDBX_txn *txn = nullptr;
  const intptr_t bitmap_bytes =
#if MDBX_ENABLE_DBI_SPARSE
      ceil_powerof2(max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT;
#else
      0;
#endif /* MDBX_ENABLE_DBI_SPARSE */
  const size_t base = sizeof(MDBX_txn) + /* GC cursor */ sizeof(cursor_couple_t);
  const size_t size =
      base + bitmap_bytes +
      max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_seqs[0]) + sizeof(txn->dbi_state[0]));

  txn = osal_calloc(1, size);
  if (unlikely(!txn))
    return txn;

  rkl_init(&txn->wr.gc.reclaimed);
  rkl_init(&txn->wr.gc.ready4reuse);
  rkl_init(&txn->wr.gc.comeback);
  txn->dbs = ptr_disp(txn, base);
  txn->cursors = ptr_disp(txn->dbs, max_dbi * sizeof(txn->dbs[0]));
  txn->dbi_seqs = ptr_disp(txn->cursors, max_dbi * sizeof(txn->cursors[0]));
  txn->dbi_state = ptr_disp(txn, size - max_dbi * sizeof(txn->dbi_state[0]));
#if MDBX_ENABLE_DBI_SPARSE
  txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes);
#endif /* MDBX_ENABLE_DBI_SPARSE */
  txn->flags = MDBX_TXN_FINISHED;
  txn->wr.retired_pages = pnl_alloc(MDBX_PNL_INITIAL);
  txn->wr.repnl = pnl_alloc(MDBX_PNL_INITIAL);
  if (unlikely(!txn->wr.retired_pages || !txn->wr.repnl)) {
    txn_basal_destroy(txn);
    txn = nullptr;
  }

  return txn;
}

__cold void txn_basal_destroy(MDBX_txn *txn) {
  txn_dpl_free(txn);
  rkl_destroy(&txn->wr.gc.reclaimed);
  rkl_destroy(&txn->wr.gc.ready4reuse);
  rkl_destroy(&txn->wr.gc.comeback);
  pnl_free(txn->wr.retired_pages);
  pnl_free(txn->wr.spilled.list);
  pnl_free(txn->wr.repnl);
  osal_free(txn);
}

static bool basal_check_overlapped(lck_t *const lck, const mdbx_pid_t pid, const uintptr_t tid) {
  const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
  for (size_t i = 0; i < snap_nreaders; ++i) {
    if (atomic_load_pid(&lck->rdt[i].pid, mo_Relaxed) == pid &&
        unlikely(atomic_load64(&lck->rdt[i].tid, mo_Relaxed) == tid)) {
      const txnid_t txnid = safe64_read(&lck->rdt[i].txnid);
      if (txnid >= MIN_TXNID && txnid <= MAX_TXNID)
        return true;
    }
  }
  return false;
}

static int basal_start_locked(MDBX_txn *txn, unsigned flags) {
  MDBX_env *const env = txn->env;
  if (unlikely(env->flags & ENV_FATAL_ERROR))
    return MDBX_PANIC;

#if defined(_WIN32) || defined(_WIN64)
  if (unlikely(!env->dxb_mmap.base))
    return MDBX_EPERM;
#endif /* Windows */

  txn->flags = flags & ~txn_rw_already_locked;
  txn->nested = nullptr;
  txn->wr.loose_pages = nullptr;
  txn->wr.loose_count = 0;
#if MDBX_ENABLE_REFUND
  txn->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
  pnl_setsize(txn->wr.retired_pages, 0);
  txn->wr.spilled.list = nullptr;
  txn->wr.spilled.least_removed = 0;
  txn->wr.gc.spent = 0;
  cASSERT0(txn, rkl_empty(&txn->wr.gc.reclaimed));
  cASSERT0(txn, rkl_empty(&txn->wr.gc.ready4reuse));
  cASSERT0(txn, rkl_empty(&txn->wr.gc.comeback));
  txn->env->gc.detent = 0;
#if MDBX_ENABLE_PGET_STAT
  txn->ops_pget = 0;
#endif /* MDBX_ENABLE_PGET_STAT */

  txn->wr.troika = meta_tap(env);
  const meta_ptr_t head = meta_recent(env, &txn->wr.troika);
  uint64_t timestamp = 0;
  /* coverity[NO_EFFECT] */
  while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") {
    int err = coherency_fetch_head(txn, head, &timestamp);
    if (likely(err == MDBX_SUCCESS))
      break;
    if (unlikely(err != MDBX_RESULT_TRUE))
      return err;
  }
  eASSERT1(env, meta_txnid(head.ptr_v) == txn->txnid);
  txn->txnid = safe64_txnid_next(txn->txnid);
  if (unlikely(txn->txnid > MAX_TXNID)) {
    ERROR("txnid overflow, raise %d", MDBX_TXN_FULL);
    return MDBX_TXN_FULL;
  }

  int err = txn_setup_primal(txn);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  eASSERT0(env, pgno2bytes(env, txn->geo.first_unallocated) <= env->dxb_mmap.current);
  eASSERT0(env, env->dxb_mmap.limit >= env->dxb_mmap.current);

  if (env->options.need_dp_limit_adjust)
    env_options_adjust_dp_limit(env);
  if ((txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) {
    err = txn_dpl_alloc(txn);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
    txn->wr.dirtyroom = txn->env->options.dp_limit;
    txn->wr.dirtylru = (MDBX_DEBUG > 0) ? UINT32_MAX / 3 - 42 : 0;
  } else {
    cASSERT0(txn, txn->wr.dirtylist == nullptr);
    txn->wr.dirtylist = nullptr;
    txn->wr.dirtyroom = MAX_PAGENO;
    txn->wr.dirtylru = 0;
  }

  eASSERT0(env, txn->wr.writemap_dirty_npages == 0);
  eASSERT0(env, txn->wr.writemap_spilled_npages == 0);
  MDBX_cursor *const gc = ptr_disp(txn, sizeof(MDBX_txn));
  err = cursor_init(gc, txn, FREE_DBI);
  if (err != MDBX_SUCCESS)
    return err;
  cASSERT0(txn, txn->cursors[FREE_DBI] == nullptr);

  dxb_sanitize_tail(env, txn);
  env->txn = txn;
  return MDBX_SUCCESS;
}

int txn_basal_start(MDBX_txn *txn, unsigned flags) {
  cASSERT0(txn, (flags & ~(txn_rw_begin_flags | MDBX_TXN_SPILLS | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS |
                           txn_rw_already_locked)) == 0);
  MDBX_env *const env = txn->env;
  cASSERT0(txn, txn == env->basal_txn);
  flags |= env->flags & (MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP);
  if ((flags & txn_rw_already_locked) == 0) {
    const uintptr_t tid = osal_thread_self();
    if (unlikely(txn->owner == tid ||
                 /* not recovery mode */ env->stuck_meta >= 0))
      return MDBX_BUSY;

    lck_t *const lck = env->lck_mmap.lck;
    if (lck && !(flags & MDBX_NOSTICKYTHREADS) && !(globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) &&
        basal_check_overlapped(lck, env->pid, tid))
      return MDBX_TXN_OVERLAPPING;

    /* Not yet touching txn == env->basal_txn, it may be active */
    jitter4testing(false);
    int err = lck_txn_lock(env, (flags & MDBX_TXN_TRY) != 0);
    if (unlikely(err != MDBX_SUCCESS))
      return err;
  }

  int err = basal_start_locked(txn, flags);
  if (unlikely(err != MDBX_SUCCESS)) {
    txn_basal_end(txn, true);
    return err;
  }

  return MDBX_SUCCESS;
}

int txn_basal_end(MDBX_txn *txn, bool unlock) {
  MDBX_env *const env = txn->env;
  cASSERT0(txn,
           !txn->parent && !txn->nested && (txn->flags & (txn_ro_both | MDBX_TXN_HAS_CHILD | MDBX_TXN_PARKED)) == 0);
  cASSERT0(txn, (txn->flags & txn_may_have_cursors) == 0);
  if ((txn->flags & (MDBX_TXN_ERROR | MDBX_TXN_FINISHED)) == 0)
    ENSURE_OBJ(env, txn->txnid > /* paranoia is appropriate here */ env->lck->cached_oldest_txnid.weak);
  dxb_sanitize_tail(env, nullptr);

  const unsigned preserved_flags = txn->flags;
  txn->flags = MDBX_TXN_FINISHED | (env->flags & (MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS));
  env->txn = nullptr;
  pnl_free(txn->wr.spilled.list);
  txn->wr.spilled.list = nullptr;
  rkl_clear_and_shrink(&txn->wr.gc.reclaimed);
  rkl_clear_and_shrink(&txn->wr.gc.ready4reuse);
  rkl_clear_and_shrink(&txn->wr.gc.comeback);

  eASSERT0(env, txn->parent == nullptr);
  pnl_shrink(&txn->wr.retired_pages);
  pnl_shrink(&txn->wr.repnl);
  txn_dpl_clear(txn);

  /* Export or close DBI handles created in this txn */
  int rc = (preserved_flags & MDBX_TXN_DIRTY) ? dbi_update(txn, !(preserved_flags & MDBX_TXN_ERROR)) : MDBX_SUCCESS;
  if (likely(unlock) || unlikely(rc != MDBX_SUCCESS)) {
    /* The writer mutex was locked in mdbx_txn_begin. */
    lck_txn_unlock(env);
  }
  return rc;
}

/* Update table root pointers */
int txn_basal_update_tbl_roots(MDBX_txn *txn) {
  cursor_couple_t cx;
  int err = MDBX_SUCCESS;
  if (txn->n_dbi > CORE_DBS) {
    err = cursor_init(&cx.outer, txn, MAIN_DBI);
    if (likely(err == MDBX_SUCCESS)) {
      cx.outer.next = txn->cursors[MAIN_DBI];
      txn->cursors[MAIN_DBI] = &cx.outer;
      TXN_FOREACH_DBI_USER(txn, i) {
        if ((txn->dbi_state[i] & DBI_DIRTY) == 0)
          continue;
        tree_t *const db = &txn->dbs[i];
        DEBUG("update main's entry for sub-db %zu, mod_txnid %" PRIaTXN " -> %" PRIaTXN, i, db->mod_txnid, txn->txnid);
        /* Может быть mod_txnid > front после коммита вложенных тразакций */
        db->mod_txnid = txn->txnid;
        MDBX_val data = {db, sizeof(tree_t)};
        /* coverity[OVERRUN] */
        err = cursor_put(&cx.outer, &txn->env->kvs[i].name, &data, N_TREE);
        if (unlikely(err != MDBX_SUCCESS))
          break;
      }
      txn->cursors[MAIN_DBI] = cx.outer.next;
    }
  }
  return err;
}

int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
  MDBX_env *const env = txn->env;
  cASSERT0(txn, txn == env->basal_txn && !txn->parent && !txn->nested);
  cASSERT0(txn, (txn->flags & MDBX_TXN_ERROR) == 0);
  if (!txn->wr.dirtylist) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
  } else {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
    cASSERT0(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == env->options.dp_limit);
  }

  if (txn->flags & txn_may_have_cursors)
    txn_done_cursors(txn);

  bool need_flush_for_nometasync = false;
  const meta_ptr_t head = meta_recent(env, &txn->wr.troika);
  const uint32_t meta_sync_txnid = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed);
  /* sync prev meta */
  if (head.is_steady && meta_sync_txnid != (uint32_t)head.txnid) {
    /* Исправление унаследованного от LMDB недочета:
     *
     * Всё хорошо, если все процессы работающие с БД не используют WRITEMAP.
     * Тогда мета-страница (обновленная, но не сброшенная на диск) будет
     * сохранена в результате fdatasync() при записи данных этой транзакции.
     *
     * Всё хорошо, если все процессы работающие с БД используют WRITEMAP
     * без MDBX_AVOID_MSYNC.
     * Тогда мета-страница (обновленная, но не сброшенная на диск) будет
     * сохранена в результате msync() при записи данных этой транзакции.
     *
     * Если же в процессах работающих с БД используется оба метода, как sync()
     * в режиме MDBX_WRITEMAP, так и записи через файловый дескриптор, то
     * становится невозможным обеспечить фиксацию на диске мета-страницы
     * предыдущей транзакции и данных текущей транзакции, за счет одной
     * sync-операцией выполняемой после записи данных текущей транзакции.
     * Соответственно, требуется явно обновлять мета-страницу, что полностью
     * уничтожает выгоду от NOMETASYNC. */
    const uint32_t txnid_dist = ((txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) ? MDBX_NOMETASYNC_LAZY_FD
                                                                                        : MDBX_NOMETASYNC_LAZY_WRITEMAP;
    /* Смысл "магии" в том, чтобы избежать отдельного вызова fdatasync()
     * или msync() для гарантированной фиксации на диске мета-страницы,
     * которая была "лениво" отправлена на запись в предыдущей транзакции,
     * но не сброшена на диск из-за активного режима MDBX_NOMETASYNC. */
    if (
#if defined(_WIN32) || defined(_WIN64)
        !env->ioring.overlapped_fd &&
#endif
        meta_sync_txnid == (uint32_t)head.txnid - txnid_dist)
      need_flush_for_nometasync = true;
    else {
      int err = meta_sync(env, head);
      if (unlikely(err != MDBX_SUCCESS)) {
        ERROR("txn-%s: error %d", "presync-meta", err);
        return err;
      }
    }
  }

  if ((!txn->wr.dirtylist || txn->wr.dirtylist->length == 0) &&
      (txn->flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_NOSYNC | MDBX_TXN_NOMETASYNC)) == 0 &&
      !need_flush_for_nometasync && !head.is_steady && !CHECKS2_ENABLED()) {
    TXN_FOREACH_DBI_ALL(txn, i) { cASSERT0(txn, !(txn->dbi_state[i] & DBI_DIRTY)); }
    /* fast completion of pure transaction */
    return MDBX_NOSUCCESS_PURE_COMMIT ? MDBX_RESULT_TRUE : MDBX_SUCCESS;
  }

  DEBUG("committing txn %" PRIaTXN " %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid,
        __Wpedantic_format_voidptr(txn), __Wpedantic_format_voidptr(env), txn->dbs[MAIN_DBI].root,
        txn->dbs[FREE_DBI].root);

  int rc = txn_basal_update_tbl_roots(txn);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  if (ts) {
    ts->prep = osal_monotime();
    ts->gc_cpu = osal_cputime(nullptr);
  }

  gcu_t gcu_ctx;
  rc = gc_put_init(txn, &gcu_ctx);
  if (likely(rc == MDBX_SUCCESS))
    rc = gc_update(txn, &gcu_ctx);

  const txnid_t commit_txnid =
#if MDBX_ENABLE_BIGFOOT
      gcu_ctx.bigfoot;
#else
      txn->txnid;
#endif /* MDBX_ENABLE_BIGFOOT */
  if (commit_txnid > txn->txnid)
    VERBOSE("use @%" PRIaTXN " (+%zu) for committing bigfoot-txn", commit_txnid, (size_t)(commit_txnid - txn->txnid));
  else
    VERBOSE("committing @%" PRIaTXN " txn", commit_txnid);
  gc_put_destroy(&gcu_ctx);

  if (ts)
    ts->gc_cpu = osal_cputime(nullptr) - ts->gc_cpu;
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  cASSERT0(txn, txn->wr.loose_count == 0);
  txn->dbs[FREE_DBI].mod_txnid = (txn->dbi_state[FREE_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[FREE_DBI].mod_txnid;
  txn->dbs[MAIN_DBI].mod_txnid = (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[MAIN_DBI].mod_txnid;

  if (ts) {
    ts->gc = osal_monotime();
    ts->audit = ts->gc;
  }
  if (CHECKS2_ENABLED()) {
    rc = audit_ex(txn, pnl_size(txn->wr.retired_pages), true);
    if (ts)
      ts->audit = osal_monotime();
    if (unlikely(rc != MDBX_SUCCESS))
      return rc;
  }

  if (txn->wr.dirtylist) {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
    cASSERT0(txn, txn->wr.loose_count == 0);

    mdbx_filehandle_t fd =
#if defined(_WIN32) || defined(_WIN64)
        env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
    (void)need_flush_for_nometasync;
#else
        (need_flush_for_nometasync || env->dsync_fd == INVALID_HANDLE_VALUE ||
         txn->wr.dirtylist->length > env->options.writethrough_threshold ||
         atomic_load64(&env->lck->unsynced_pages, mo_Relaxed))
            ? env->lazy_fd
            : env->dsync_fd;
#endif /* Windows */

    iov_ctx_t write_ctx;
    rc = iov_init(txn, &write_ctx, txn->wr.dirtylist->length, txn->wr.dirtylist->pages_including_loose, fd, false);
    if (unlikely(rc != MDBX_SUCCESS)) {
      ERROR("txn-%s: error %d", "iov-init", rc);
      return rc;
    }

    rc = txn_write(txn, &write_ctx);
    if (unlikely(rc != MDBX_SUCCESS)) {
      ERROR("txn-%s: error %d", "write", rc);
      return rc;
    }
  } else {
    cASSERT0(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
    env->lck->unsynced_pages.weak += txn->wr.writemap_dirty_npages;
    if (!env->lck->eoos_timestamp.weak)
      env->lck->eoos_timestamp.weak = osal_monotime();
  }

  /* TODO: use ctx.flush_begin & ctx.flush_end for range-sync */
  if (ts)
    ts->write = osal_monotime();

  meta_t meta;
  memcpy(meta.magic_and_version, head.ptr_c->magic_and_version, 8);
  meta.reserve16 = head.ptr_c->reserve16;
  meta.validator_id = head.ptr_c->validator_id;
  meta.extra_pagehdr = head.ptr_c->extra_pagehdr;
  unaligned_poke_u64(4, meta.pages_retired,
                     unaligned_peek_u64(4, head.ptr_c->pages_retired) + pnl_size(txn->wr.retired_pages));
  meta.geometry = txn->geo;
  meta.trees.gc = txn->dbs[FREE_DBI];
  meta.trees.main = txn->dbs[MAIN_DBI];
  meta.canary = txn->canary;
  memcpy(&meta.dxbid, &head.ptr_c->dxbid, sizeof(meta.dxbid));

  meta.unsafe_sign = DATASIGN_NONE;
  meta_set_txnid(env, &meta, commit_txnid);

  rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, &txn->wr.troika);

  if (ts)
    ts->sync = osal_monotime();
  if (unlikely(rc != MDBX_SUCCESS)) {
    env->flags |= ENV_FATAL_ERROR;
    ERROR("txn-%s: error %d", "sync", rc);
    return rc;
  }

  return MDBX_SUCCESS;
}

int txn_basal_checkpoint(MDBX_txn *txn, MDBX_txn_flags_t weakening_durability, struct commit_timestamp *ts) {
  const unsigned preserved_flags = txn->flags & txn_rw_begin_flags;
  /* void *const preserved_context = txn->userctx; */
  txn->flags |= weakening_durability & (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_SYNC_DURABLE);
  int rc = txn_basal_commit(txn, ts);
  if (likely(rc == MDBX_SUCCESS)) {
    rc = txn_basal_end(txn, false);
    if (likely(rc == MDBX_SUCCESS)) {
      /* txn->userctx = preserved_context; */
      return txn_basal_start(txn, preserved_flags | txn_rw_already_locked);
    }
  }

  int err = txn_basal_end(txn, true);
  return (err == MDBX_SUCCESS) ? rc : err;
}

int txn_basal_rollback(MDBX_txn *txn) {
  const unsigned preserved_flags = txn->flags & txn_rw_begin_flags;
  /* void *const preserved_context = txn->userctx; */
  int rc = txn_basal_end(txn, false);
  if (likely(rc == MDBX_SUCCESS))
    /* txn->userctx = preserved_context; */
    rc = txn_basal_start(txn, preserved_flags | txn_rw_already_locked);
  return rc;
}

/* Merge pageset of the nested transaction into parent */
static void nested_merge(MDBX_txn *const parent, MDBX_txn *const nested, const size_t parent_retired_len) {
  cASSERT0(nested, (nested->flags & MDBX_WRITEMAP) == 0);
  dpl_t *const src = txn_dpl_sort(nested);

  /* Remove refunded pages from parent's dirty list */
  dpl_t *const dst = txn_dpl_sort(parent);
  if (MDBX_ENABLE_REFUND) {
    size_t n = dst->length;
    while (n && dst->items[n].pgno >= parent->geo.first_unallocated) {
      const unsigned npages = dpl_npages(dst, n);
      page_shadow_release(nested->env, dst->items[n].ptr, npages);
      --n;
    }
    parent->wr.dirtyroom += dst->sorted - n;
    dst->sorted = dpl_setlen(dst, n);
    cASSERT0(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length ==
                         (parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit));
  }

  /* Remove reclaimed pages from parent's dirty list.
   * Here the nested->wr.repnl was already moved into parent->wr.repnl
   * and space was reserved via retired_delta. */
  const pnl_t reclaimed_list = parent->wr.repnl;
  txn_dpl_sift(parent, reclaimed_list, false);

  /* Move retired pages from parent's dirty & spilled list to reclaimed */
  size_t r, w, d, s, l;
  for (r = w = parent_retired_len; ++r <= pnl_size(parent->wr.retired_pages);) {
    const pgno_t pgno = parent->wr.retired_pages[r];
    const size_t di = txn_dpl_exist(parent, pgno);
    const size_t si = !di ? spill_search(parent, pgno) : 0;
    unsigned npages;
    const char *kind;
    if (di) {
      page_t *dp = dst->items[di].ptr;
      cASSERT0(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_SPILLED)) == 0);
      npages = dpl_npages(dst, di);
      page_wash(parent, di, dp, npages);
      kind = "dirty";
      l = 1;
      if (unlikely(npages > l)) {
        /* OVERFLOW-страница могла быть переиспользована по частям. Тогда
         * в retired-списке может быть только начало последовательности,
         * а остаток растащен по dirty, spilled и reclaimed спискам. Поэтому
         * переносим в reclaimed с проверкой на обрыв последовательности.
         * В любом случае, все осколки будут учтены и отфильтрованы, т.е. если
         * страница была разбита на части, то важно удалить dirty-элемент,
         * а все осколки будут учтены отдельно. */

        /* Список retired страниц не сортирован, но для ускорения сортировки
         * дополняется в соответствии с MDBX_PNL_ASCENDING */
#if MDBX_PNL_ASCENDING
        const size_t len = pnl_size(parent->wr.retired_pages);
        while (r < len && parent->wr.retired_pages[r + 1] == pgno + l) {
          ++r;
          if (++l == npages)
            break;
        }
#else
        while (w > parent_retired_len && parent->wr.retired_pages[w - 1] == pgno + l) {
          --w;
          if (++l == npages)
            break;
        }
#endif
      }
    } else if (unlikely(si)) {
      l = npages = 1;
      spill_remove(parent, si, 1);
      kind = "spilled";
    } else {
      parent->wr.retired_pages[++w] = pgno;
      continue;
    }

    DEBUG("reclaim retired parent's %u -> %zu %s page %" PRIaPGNO, npages, l, kind, pgno);
    /* The space for additions to parent->wr.repnl was already reserverd via the retired_delta. */
    int err = pnl_insert_span(&parent->wr.repnl, pgno, l);
    ENSURE_OBJ(nested, err == MDBX_SUCCESS);
  }
  pnl_setsize(parent->wr.retired_pages, w);

  /* Filter-out parent spill list */
  if (parent->wr.spilled.list && pnl_size(parent->wr.spilled.list) > 0) {
    const pnl_t sl = spill_purge(parent);
    size_t len = pnl_size(sl);
    if (len) {
      /* Remove refunded pages from parent's spill list */
      if (MDBX_ENABLE_REFUND && MDBX_PNL_MOST(sl) >= (parent->geo.first_unallocated << 1)) {
#if MDBX_PNL_ASCENDING
        size_t i = pnl_size(sl);
        ASSERT(MDBX_PNL_MOST(sl) == MDBX_PNL_LAST(sl));
        do {
          if ((sl[i] & 1) == 0)
            DEBUG("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1);
          i -= 1;
        } while (i && sl[i] >= (parent->geo.first_unallocated << 1));
        pnl_setsize(sl, i);
#else
        ASSERT(MDBX_PNL_MOST(sl) == MDBX_PNL_FIRST(sl));
        size_t i = 0;
        do {
          ++i;
          if ((sl[i] & 1) == 0)
            DEBUG("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1);
        } while (i < len && sl[i + 1] >= (parent->geo.first_unallocated << 1));
        pnl_setsize(sl, len -= i);
        memmove(sl + 1, sl + 1 + i, len * sizeof(sl[0]));
#endif
      }
      tASSERT1(nested, pnl_check_allocated(sl, (size_t)parent->geo.first_unallocated << 1));

      /* Remove reclaimed pages from parent's spill list */
      s = pnl_size(sl), r = pnl_size(reclaimed_list);
      /* Scanning from end to begin */
      while (s && r) {
        if (sl[s] & 1) {
          --s;
          continue;
        }
        const pgno_t spilled_pgno = sl[s] >> 1;
        const pgno_t reclaimed_pgno = reclaimed_list[r];
        if (reclaimed_pgno != spilled_pgno) {
          const bool cmp = MDBX_PNL_ORDERED(spilled_pgno, reclaimed_pgno);
          s -= !cmp;
          r -= cmp;
        } else {
          DEBUG("remove reclaimed parent's spilled page %" PRIaPGNO, reclaimed_pgno);
          spill_remove(parent, s, 1);
          --s;
          --r;
        }
      }

      /* Remove anything in our dirty list from parent's spill list */
      /* Scanning spill list in descend order */
      const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1;
      s = MDBX_PNL_ASCENDING ? pnl_size(sl) : 1;
      d = src->length;
      while (d && (MDBX_PNL_ASCENDING ? s > 0 : s <= pnl_size(sl))) {
        if (sl[s] & 1) {
          s += step;
          continue;
        }
        const pgno_t spilled_pgno = sl[s] >> 1;
        const pgno_t dirty_pgno_form = src->items[d].pgno;
        const unsigned npages = dpl_npages(src, d);
        const pgno_t dirty_pgno_to = dirty_pgno_form + npages;
        if (dirty_pgno_form > spilled_pgno) {
          --d;
          continue;
        }
        if (dirty_pgno_to <= spilled_pgno) {
          s += step;
          continue;
        }

        DEBUG("remove dirtied parent's spilled %u page %" PRIaPGNO, npages, dirty_pgno_form);
        spill_remove(parent, s, 1);
        s += step;
      }

      /* Squash deleted pagenums if we deleted any */
      spill_purge(parent);
    }
  }

  /* Remove anything in our spill list from parent's dirty list */
  if (nested->wr.spilled.list) {
    tASSERT1(nested, pnl_check_allocated(nested->wr.spilled.list, (size_t)parent->geo.first_unallocated << 1));
    txn_dpl_sift(parent, nested->wr.spilled.list, true);
    cASSERT0(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length ==
                         (parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit));
  }

  /* Find length of merging our dirty list with parent's and release
   * filter-out pages */
  for (l = 0, d = dst->length, s = src->length; d > 0 && s > 0;) {
    page_t *sp = src->items[s].ptr;
    cASSERT0(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_LOOSE | P_SPILLED)) == 0);
    const unsigned s_npages = dpl_npages(src, s);
    const pgno_t s_pgno = src->items[s].pgno;

    page_t *dp = dst->items[d].ptr;
    cASSERT0(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_SPILLED)) == 0);
    const unsigned d_npages = dpl_npages(dst, d);
    const pgno_t d_pgno = dst->items[d].pgno;

    if (d_pgno >= s_pgno + s_npages) {
      --d;
      ++l;
    } else if (d_pgno + d_npages <= s_pgno) {
      if (sp->flags != P_LOOSE) {
        sp->txnid = parent->front_txnid;
        sp->flags &= ~P_SPILLED;
      }
      --s;
      ++l;
    } else {
      dst->items[d--].ptr = nullptr;
      page_shadow_release(nested->env, dp, d_npages);
    }
  }
  ASSERT(dst->sorted == dst->length);
  cASSERT0(parent, dst->detent >= l + d + s);
  dst->sorted = l + d + s; /* the merged length */

  while (s > 0) {
    page_t *sp = src->items[s].ptr;
    cASSERT0(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_LOOSE | P_SPILLED)) == 0);
    if (sp->flags != P_LOOSE) {
      sp->txnid = parent->front_txnid;
      sp->flags &= ~P_SPILLED;
    }
    --s;
  }

  /* Merge our dirty list into parent's, i.e. merge(dst, src) -> dst */
  if (dst->sorted >= dst->length) {
    /* from end to begin with dst extending */
    for (l = dst->sorted, s = src->length, d = dst->length; s > 0 && d > 0;) {
      if (unlikely(l <= d)) {
        /* squash to get a gap of free space for merge */
        for (r = w = 1; r <= d; ++r)
          if (dst->items[r].ptr) {
            if (w != r) {
              dst->items[w] = dst->items[r];
              dst->items[r].ptr = nullptr;
            }
            ++w;
          }
        VERBOSE("squash to begin for extending-merge %zu -> %zu", d, w - 1);
        d = w - 1;
        continue;
      }
      ASSERT(l > d);
      if (dst->items[d].ptr) {
        dst->items[l--] = (dst->items[d].pgno > src->items[s].pgno) ? dst->items[d--] : src->items[s--];
      } else
        --d;
    }
    if (s > 0) {
      ASSERT(l == s);
      while (d > 0) {
        ASSERT(dst->items[d].ptr == nullptr);
        --d;
      }
      do {
        ASSERT(l > 0);
        dst->items[l--] = src->items[s--];
      } while (s > 0);
    } else {
      ASSERT(l == d);
      while (l > 0) {
        ASSERT(dst->items[l].ptr != nullptr);
        --l;
      }
    }
  } else {
    /* from begin to end with shrinking (a lot of new large/overflow pages) */
    for (l = s = d = 1; s <= src->length && d <= dst->length;) {
      if (unlikely(l >= d)) {
        /* squash to get a gap of free space for merge */
        for (r = w = dst->length; r >= d; --r)
          if (dst->items[r].ptr) {
            if (w != r) {
              dst->items[w] = dst->items[r];
              dst->items[r].ptr = nullptr;
            }
            --w;
          }
        VERBOSE("squash to end for shrinking-merge %zu -> %zu", d, w + 1);
        d = w + 1;
        continue;
      }
      ASSERT(l < d);
      if (dst->items[d].ptr) {
        dst->items[l++] = (dst->items[d].pgno < src->items[s].pgno) ? dst->items[d++] : src->items[s++];
      } else
        ++d;
    }
    if (s <= src->length) {
      ASSERT(dst->sorted - l == src->length - s);
      while (d <= dst->length) {
        ASSERT(dst->items[d].ptr == nullptr);
        --d;
      }
      do {
        ASSERT(l <= dst->sorted);
        dst->items[l++] = src->items[s++];
      } while (s <= src->length);
    } else {
      ASSERT(dst->sorted - l == dst->length - d);
      while (l <= dst->sorted) {
        ASSERT(l <= d && d <= dst->length && dst->items[d].ptr);
        dst->items[l++] = dst->items[d++];
      }
    }
  }
  parent->wr.dirtyroom -= dst->sorted - dst->length;
  ASSERT(parent->wr.dirtyroom <= parent->env->options.dp_limit);
  dpl_setlen(dst, dst->sorted);
  parent->wr.dirtylru = nested->wr.dirtylru;

  /* В текущем понимании выгоднее пересчитать кол-во страниц,
   * чем подмешивать лишние ветвления и вычисления в циклы выше. */
  dst->pages_including_loose = 0;
  for (r = 1; r <= dst->length; ++r)
    dst->pages_including_loose += dpl_npages(dst, r);

  tASSERT1(parent, txn_dpl_check(parent));
  dpl_setlen(src, 0);
  txn_dpl_free(nested);

  if (nested->wr.spilled.list) {
    if (parent->wr.spilled.list) {
      /* Must not fail since space was preserved above. */
      pnl_merge(parent->wr.spilled.list, nested->wr.spilled.list);
      pnl_free(nested->wr.spilled.list);
    } else {
      parent->wr.spilled.list = nested->wr.spilled.list;
      parent->wr.spilled.least_removed = nested->wr.spilled.least_removed;
    }
    tASSERT1(parent, txn_dpl_check(parent));
  }

  if (parent->wr.spilled.list) {
    ASSERT(pnl_check_allocated(parent->wr.spilled.list, (size_t)parent->geo.first_unallocated << 1));
    if (pnl_size(parent->wr.spilled.list))
      parent->flags |= MDBX_TXN_SPILLS;
  }
}

static int nested_start(MDBX_txn *const nested, MDBX_txn *parent) {
  tASSERT1(parent, txn_dpl_check(parent));

  nested->txnid = parent->txnid;
  nested->front_txnid = parent->front_txnid + 1;
  nested->canary = parent->canary;
  parent->flags |= MDBX_TXN_HAS_CHILD;
  parent->nested = nested;
  nested->parent = parent;
  nested->env->txn = nested;
  nested->owner = parent->owner;
  nested->wr.troika = parent->wr.troika;

  const size_t len = pnl_size(parent->wr.repnl) + parent->wr.loose_count;
  cASSERT0(nested, !nested->wr.repnl);
  nested->wr.repnl = pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL);
  if (unlikely(!nested->wr.repnl))
    return LOG_IFERR(MDBX_ENOMEM);

#if MDBX_ENABLE_DBI_SPARSE
  nested->dbi_sparse = parent->dbi_sparse;
#endif /* MDBX_ENABLE_DBI_SPARSE */
  nested->dbi_seqs = parent->dbi_seqs;
  nested->geo = parent->geo;

  int err = txn_dpl_alloc(nested);
  if (unlikely(err != MDBX_SUCCESS))
    return LOG_IFERR(err);

  /* Move loose pages to reclaimed list */
  if (parent->wr.loose_count) {
    do {
      page_t *lp = parent->wr.loose_pages;
      cASSERT0(parent, lp->flags == P_LOOSE);
      err = pnl_insert_span(&parent->wr.repnl, lp->pgno, 1);
      if (unlikely(err != MDBX_SUCCESS))
        return LOG_IFERR(err);
      MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
      VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
      parent->wr.loose_pages = page_next(lp);
      /* Remove from dirty list */
      page_wash(parent, txn_dpl_exist(parent, lp->pgno), lp, 1);
    } while (parent->wr.loose_pages);
    parent->wr.loose_count = 0;
#if MDBX_ENABLE_REFUND
    parent->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
    tASSERT1(parent, txn_dpl_check(parent));
  }
#if MDBX_ENABLE_REFUND
  nested->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
  nested->wr.dirtyroom = parent->wr.dirtyroom;
  nested->wr.dirtylru = parent->wr.dirtylru;
#if MDBX_ENABLE_PGET_STAT
  nested->ops_pget = 0;
#endif /* MDBX_ENABLE_PGET_STAT */

  txn_dpl_sort(parent);
  if (parent->wr.spilled.list)
    spill_purge(parent);

  cASSERT0(nested, pnl_alloclen(nested->wr.repnl) >= pnl_size(parent->wr.repnl));
  memcpy(nested->wr.repnl, parent->wr.repnl, MDBX_PNL_SIZEOF(parent->wr.repnl));
  /* coverity[ASSERT_SIDE_EFFECT] */
  tASSERT1(nested, pnl_check_allocated(nested->wr.repnl, (nested->geo.first_unallocated /* LY: intentional assignment
                                                                             here, only for assertion */
                                                          = parent->geo.first_unallocated) -
                                                             MDBX_ENABLE_REFUND));

  nested->wr.gc.spent = parent->wr.gc.spent;
  err = rkl_copy(&parent->wr.gc.reclaimed, &nested->wr.gc.reclaimed);
  if (unlikely(err != MDBX_SUCCESS))
    return err;
  err = rkl_copy(&parent->wr.gc.ready4reuse, &nested->wr.gc.ready4reuse);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  nested->wr.retired_pages = parent->wr.retired_pages;
  parent->wr.retired_pages = (void *)(intptr_t)pnl_size(parent->wr.retired_pages);
  nested->cursors[FREE_DBI] = nullptr;
  nested->cursors[MAIN_DBI] = nullptr;
  nested->dbi_state[FREE_DBI] = parent->dbi_state[FREE_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY);
  nested->dbi_state[MAIN_DBI] = parent->dbi_state[MAIN_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY);
  memcpy(nested->dbs, parent->dbs, sizeof(nested->dbs[0]) * CORE_DBS);
  if ((nested->n_dbi = parent->n_dbi) > CORE_DBS)
    memset(nested->dbi_state + CORE_DBS, 0, nested->n_dbi - CORE_DBS);

  cASSERT0(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length ==
                       (parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit));
  cASSERT0(nested, nested->wr.dirtyroom + nested->wr.dirtylist->length ==
                       (nested->parent ? nested->parent->wr.dirtyroom : nested->env->options.dp_limit));
  return txn_shadow_cursors(parent, MAIN_DBI);
}

int txn_nested_create(MDBX_txn *parent, bool readonly) {
  if (unlikely(parent->flags & MDBX_WRITEMAP)) {
    ERROR("%s mode is incompatible with nested transactions", "MDBX_WRITEMAP");
    return MDBX_INCOMPATIBLE;
  }

  unsigned flags = parent->flags & (MDBX_TXN_SPILLS | MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP);
  if (readonly)
    flags |= txn_ro_nested;

  if (parent->env->options.spill_parent4child_denominator && (flags & txn_ro_nested) == 0) {
    /* Spill dirty-pages of parent to provide dirtyroom for child txn */
    int err =
        txn_spill(parent, nullptr, parent->wr.dirtylist->length / parent->env->options.spill_parent4child_denominator);
    if (unlikely(err != MDBX_SUCCESS))
      return LOG_IFERR(err);
  }
  tASSERT2(parent, audit_ex(parent, 0, false) == 0);

  MDBX_txn *const nested = txn_alloc(flags, parent->env);
  if (unlikely(!nested))
    return LOG_IFERR(MDBX_ENOMEM);

  rkl_init(&nested->wr.gc.comeback);
  int rc = nested_start(nested, parent);
  if (unlikely(rc != MDBX_SUCCESS))
    txn_nested_abort(nested);
  return rc;
}

static int nested_undo(MDBX_txn *nested) {
  cASSERT0(nested, !nested->nested && !(nested->flags & MDBX_TXN_HAS_CHILD));
  cASSERT0(nested, rkl_empty(&nested->wr.gc.comeback));
  if (nested->flags & txn_may_have_cursors)
    txn_done_cursors(nested);
  if (nested->flags & MDBX_TXN_DIRTY)
    dbi_update(nested, false);

  MDBX_txn *const parent = nested->parent;
  if (nested->wr.retired_pages) {
    cASSERT0(parent, pnl_size(nested->wr.retired_pages) >= (uintptr_t)parent->wr.retired_pages);
    pnl_setsize(nested->wr.retired_pages, (uintptr_t)parent->wr.retired_pages);
    parent->wr.retired_pages = nested->wr.retired_pages;
  }

  nested->flags = MDBX_TXN_FINISHED;
  MDBX_env *const env = nested->env;
  if (unlikely(parent->geo.upper != nested->geo.upper || parent->geo.now != nested->geo.now) &&
      !(parent->flags & MDBX_TXN_ERROR) && !(env->flags & ENV_FATAL_ERROR)) {
    /* undo resize performed by nested txn */
    int err = dxb_resize(env, parent->geo.first_unallocated, parent->geo.now, parent->geo.upper, impilict_shrink);
    if (err == MDBX_EPERM) {
      /* unable undo resize (it is regular for Windows),
       * therefore promote size changes from nested to the parent txn */
      WARNING("unable undo resize performed by nested txn, promote to "
              "the parent (%u->%u, %u->%u)",
              nested->geo.upper, parent->geo.now, nested->geo.upper, parent->geo.upper);
      parent->geo.now = nested->geo.upper;
      parent->flags |= MDBX_TXN_DIRTY;
    } else if (unlikely(err != MDBX_SUCCESS)) {
      ERROR("error %d while undo resize performed by nested txn, fail the parent", err);
      mdbx_txn_break(env->basal_txn);
      parent->flags |= MDBX_TXN_ERROR;
      if (!env->dxb_mmap.base)
        env->flags |= ENV_FATAL_ERROR;
    }
  }
  return MDBX_SUCCESS;
}

static void nested_free(MDBX_txn *nested) {
  cASSERT0(nested, !(nested->flags & txn_may_have_cursors));
  MDBX_env *const env = nested->env;
  MDBX_txn *const parent = nested->parent;
  cASSERT0(parent, parent->flags & MDBX_TXN_HAS_CHILD);
  env->txn = parent;
  parent->nested = nullptr;
  parent->flags -= MDBX_TXN_HAS_CHILD;
  nested->signature = 0;
  nested->owner = 0;

  cASSERT0(nested, rkl_empty(&nested->wr.gc.comeback));
  rkl_destroy(&nested->wr.gc.reclaimed);
  rkl_destroy(&nested->wr.gc.ready4reuse);

  txn_dpl_free(nested);
  pnl_free(nested->wr.repnl);
  osal_free(nested);

  tASSERT1(parent, txn_dpl_check(parent));
  tASSERT2(parent, audit_ex(parent, 0, false) == 0);
}

int txn_nested_abort(MDBX_txn *nested) {
  tASSERT0(nested, nested != nested->env->basal_txn);
  tASSERT0(nested, nested->parent->nested == nested && (nested->parent->flags & MDBX_TXN_HAS_CHILD) != 0);
  tASSERT1(nested, txn_dpl_check(nested));
  tASSERT1(nested, pnl_check_allocated(nested->wr.repnl, nested->geo.first_unallocated - MDBX_ENABLE_REFUND));
  tASSERT1(nested, memcmp(&nested->wr.troika, &nested->parent->wr.troika, sizeof(troika_t)) == 0);

  int rc = nested_undo(nested);
  nested_free(nested);
  return rc;
}

static int nested_join(MDBX_txn *nested, struct commit_timestamp *ts) {
  MDBX_env *const env = nested->env;
  MDBX_txn *const parent = nested->parent;
  tASSERT2(nested, audit_ex(nested, 0, false) == 0);
  eASSERT0(env, nested != env->basal_txn);
  eASSERT0(env, parent->nested == nested && (parent->flags & MDBX_TXN_HAS_CHILD) != 0);
  eASSERT1(env, txn_dpl_check(nested));
  tASSERT1(nested, pnl_check_allocated(nested->wr.repnl, nested->geo.first_unallocated - MDBX_ENABLE_REFUND));
  tASSERT1(nested, memcmp(&nested->wr.troika, &parent->wr.troika, sizeof(troika_t)) == 0);
  tASSERT0(nested, rkl_empty(&nested->wr.gc.comeback));

  //-------------------------------------------------------------------------
  // Preserve space for page lists in the parent transaction.

  const size_t parent_retired_len = (uintptr_t)parent->wr.retired_pages;
  if (nested->flags & MDBX_TXN_DIRTY) {
    tASSERT0(nested, parent_retired_len <= pnl_size(nested->wr.retired_pages));
    const size_t retired_delta = pnl_size(nested->wr.retired_pages) - parent_retired_len;
    if (retired_delta) {
      int err = pnl_need(&nested->wr.repnl, retired_delta);
      if (unlikely(err != MDBX_SUCCESS))
        return err;
    }

    if (nested->wr.spilled.list) {
      if (parent->wr.spilled.list) {
        int err = pnl_need(&parent->wr.spilled.list, pnl_size(nested->wr.spilled.list));
        if (unlikely(err != MDBX_SUCCESS))
          return err;
      }
      spill_purge(nested);
    }

    if (unlikely(nested->wr.dirtylist->length + parent->wr.dirtylist->length > parent->wr.dirtylist->detent &&
                 !txn_dpl_reserve(parent, nested->wr.dirtylist->length + parent->wr.dirtylist->length))) {
      return MDBX_ENOMEM;
    }
  }

  //-------------------------------------------------------------------------
  // No further failures should be allowed.

  if (nested->flags & txn_may_have_cursors)
    /* Merge our cursors into parent's and close them */
    txn_done_cursors(nested);

  parent->wr.retired_pages = nested->wr.retired_pages;
  nested->wr.retired_pages = nullptr;

  pnl_free(parent->wr.repnl);
  parent->wr.repnl = nested->wr.repnl;
  nested->wr.repnl = nullptr;
  parent->wr.gc.spent = nested->wr.gc.spent;
  rkl_destructive_move(&nested->wr.gc.reclaimed, &parent->wr.gc.reclaimed);
  rkl_destructive_move(&nested->wr.gc.ready4reuse, &parent->wr.gc.ready4reuse);
  tASSERT0(nested, rkl_empty(&nested->wr.gc.comeback));

  /* Update parent's DBs array */
  eASSERT0(env, parent->n_dbi == nested->n_dbi);
  TXN_FOREACH_DBI_ALL(nested, dbi) {
    if (nested->dbi_state[dbi] != (parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY))) {
      eASSERT0(env, (nested->dbi_state[dbi] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY)) != 0 ||
                        (nested->dbi_state[dbi] | DBI_STALE) ==
                            (parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY)));
      parent->dbs[dbi] = nested->dbs[dbi];
      /* preserve parent's status */
      const uint8_t state = nested->dbi_state[dbi] | (parent->dbi_state[dbi] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY));
      DEBUG("dbi %zu dbi-state %s 0x%02x -> 0x%02x", dbi, (parent->dbi_state[dbi] != state) ? "update" : "still",
            parent->dbi_state[dbi], state);
      parent->dbi_state[dbi] = state;
    }
  }

  if (ts)
    ts->prep = osal_monotime();

  if (nested->flags & MDBX_TXN_DIRTY) {
    parent->geo = nested->geo;
    parent->canary = nested->canary;
    parent->flags |= MDBX_TXN_DIRTY;

    /* Move loose pages to parent */
#if MDBX_ENABLE_REFUND
    parent->wr.loose_refund_wl = nested->wr.loose_refund_wl;
#endif /* MDBX_ENABLE_REFUND */
    parent->wr.loose_count = nested->wr.loose_count;
    parent->wr.loose_pages = nested->wr.loose_pages;

    nested_merge(parent, nested, parent_retired_len);

#if MDBX_ENABLE_REFUND
    txn_refund(parent);
    if (CHECKS1_ENABLED()) {
      /* Check parent's loose pages not suitable for refund */
      for (page_t *lp = parent->wr.loose_pages; lp; lp = page_next(lp)) {
        tASSERT0(parent, lp->pgno < parent->wr.loose_refund_wl && lp->pgno + 1 < parent->geo.first_unallocated);
        MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
        VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
      }
      /* Check parent's reclaimed pages not suitable for refund */
      if (pnl_size(parent->wr.repnl))
        tASSERT0(parent, MDBX_PNL_MOST(parent->wr.repnl) + 1 < parent->geo.first_unallocated);
    }
#endif /* MDBX_ENABLE_REFUND */
  } else {
    VERBOSE("fast-complete pure nested txn %" PRIaTXN, nested->txnid);

    tASSERT1(nested, memcmp(&parent->geo, &nested->geo, sizeof(parent->geo)) == 0);
    tASSERT1(nested, memcmp(&parent->canary, &nested->canary, sizeof(parent->canary)) == 0);
    cASSERT0(nested, !nested->wr.spilled.list || pnl_size(nested->wr.spilled.list) == 0);
    cASSERT0(nested, nested->wr.loose_count == 0);
  }

  nested->flags = MDBX_TXN_FINISHED;
  tASSERT2(parent, audit_ex(parent, 0, false) == 0);
  return MDBX_SUCCESS;
}

int txn_nested_commit(MDBX_txn *nested, struct commit_timestamp *ts) {
  int rc = nested_join(nested, ts);
  if (unlikely(rc != MDBX_SUCCESS))
    nested_undo(nested);
  nested_free(nested);
  return rc;
}

int txn_nested_checkpoint(MDBX_txn *nested, struct commit_timestamp *ts) {
  MDBX_txn *parent = nested->parent;
  if (unlikely(!parent || parent->nested != nested || parent->env != nested->env)) {
    ERROR("attempt to %s %s txn %p", "commit", "strange nested", __Wpedantic_format_voidptr(nested));
    return MDBX_PROBLEM;
  }

  unsigned flags = nested->flags & (txn_rw_begin_flags | MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | MDBX_TXN_RDONLY);
  int rc = nested_join(nested, ts);
  if (likely(rc == MDBX_SUCCESS)) {
    nested->flags = flags | (parent->flags & MDBX_TXN_SPILLS);
    nested->wr.loose_count = 0;
    nested->wr.loose_pages = nullptr;
    rc = nested_start(nested, parent);
    cASSERT0(nested, nested->env->txn == nested);
  }
  if (unlikely(rc != MDBX_SUCCESS))
    txn_nested_abort(nested);
  return rc;
}

MDBX_txn *txn_nested_fakero_begin(MDBX_txn *parent) {
  cASSERT0(parent, (parent->flags & (txn_ro_nested | MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD)) == 0);
  if (!MDBX_ENABLE_FAKE_NESTED_READONLY_TRANSACTIONS)
    return nullptr;
  parent->wr.preserve_parent_userctx = parent->userctx;
  parent->flags |= txn_ro_nested;
  return parent;
}

int txn_nested_fakero_end(MDBX_txn *nested) {
  cASSERT0(nested, nested->flags & txn_ro_nested);
  if (!MDBX_ENABLE_FAKE_NESTED_READONLY_TRANSACTIONS)
    return MDBX_BAD_TXN;
  nested->userctx = nested->wr.preserve_parent_userctx;
  nested->flags -= txn_ro_nested;
  return MDBX_SUCCESS;
}

int txn_nested_rollback(MDBX_txn *nested) {
  cASSERT0(nested, nested->flags & txn_ro_nested);
  cASSERT0(nested, rkl_empty(&nested->wr.gc.comeback));
  MDBX_txn *parent = nested->parent;
  if (unlikely(!parent || parent->nested != nested || parent->env != nested->env)) {
    ERROR("attempt to %s %s txn %p", "rollback", "strange nested", __Wpedantic_format_voidptr(nested));
    return MDBX_PROBLEM;
  }

  if (unlikely(F_ISSET(nested->flags, txn_ro_nested | MDBX_TXN_DIRTY))) {
    ERROR("unable to %s %s txn %p", "rollback", "dirty nested fake-readonly", __Wpedantic_format_voidptr(nested));
    return MDBX_BAD_TXN;
  }

  const unsigned preserved_flags = nested->flags & txn_ro_nested;
  int rc = nested_undo(nested);
  if (likely(rc == MDBX_SUCCESS))
    rc = nested_start(nested, parent);
  if (likely(rc == MDBX_SUCCESS))
    nested->flags |= preserved_flags;
  else
    txn_nested_abort(nested);
  return rc;
}

static inline mdbx_pid_t ro_slot_pid(const reader_slot_t *slot) { return slot->pid.weak; }

static inline uint64_t ro_slot_tid(const reader_slot_t *slot) { return slot->tid.weak; }

static inline txnid_t ro_slot_txnid(const reader_slot_t *slot) { return slot->txnid.weak; }

MDBX_MAYBE_UNUSED static inline bool is_ro_txn(MDBX_txn *txn) {
  return (txn->flags & (txn_ro_flat | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD)) == txn_ro_flat &&
         !txn->parent;
}

static __noinline int ro_slot_mismatch_pid(MDBX_txn *txn, const mdbx_pid_t snap_slot_pid) {
  mdbx_pid_t current_pid = osal_getpid();
  if (current_pid == txn->env->registered_reader_pid)
    ERROR("readonly-transaction %p mvcc %" PRIu64 " was unexpectedly ousted (expectd pid %zu, seen %zu)",
          __Wpedantic_format_voidptr(txn), txn->txnid, (size_t)current_pid, (size_t)snap_slot_pid);
  else
    NOTICE("ignore pid mismatch of transaction %p mvcc %" PRIu64 "reader-slot after fork",
           __Wpedantic_format_voidptr(txn), txn->txnid);

  return (current_pid != snap_slot_pid) ? MDBX_BAD_RSLOT : MDBX_SUCCESS;
}

static inline int ro_slot_checkpid(MDBX_txn *txn) {
  const mdbx_pid_t snap_slot_pid = ro_slot_pid(txn->ro.slot);
  if (likely(snap_slot_pid == txn->env->registered_reader_pid)) {
    tASSERT0(txn, snap_slot_pid == osal_getpid());
    if ((txn->flags & MDBX_TXN_FINISHED) == 0)
      cASSERT0(txn, ro_slot_tid(txn->ro.slot) == ((txn->env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()) ||
                        (ro_slot_tid(txn->ro.slot) >= MDBX_TID_TXN_OUSTED &&
                         (txn->flags & (MDBX_TXN_PARKED | MDBX_TXN_OUSTED))));
    return MDBX_SUCCESS;
  }
  return ro_slot_mismatch_pid(txn, snap_slot_pid);
}

static int ro_slot_alloc(MDBX_txn *txn) {
  tASSERT0(txn, is_ro_txn(txn) && !txn->ro.slot);
  MDBX_env *const env = txn->env;
  if (unlikely(!env->lck_mmap.lck))
    return MDBX_SUCCESS;

  if (env->flags & ENV_TXKEY) {
    eASSERT0(env, !(env->flags & MDBX_NOSTICKYTHREADS));
    reader_slot_t *slot = thread_rthc_get(env->me_txkey);
    if (likely(slot)) {
      const mdbx_pid_t snap_slot_pid = ro_slot_pid(slot);
      tASSERT1(txn, env->registered_reader_pid == osal_getpid());
      if (likely(snap_slot_pid == env->registered_reader_pid && ro_slot_txnid(slot) >= SAFE64_INVALID_THRESHOLD)) {
        tASSERT1(txn, ro_slot_tid(slot) == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()));
        txn->ro.slot = slot;
        return MDBX_SUCCESS;
      }
      if (unlikely(snap_slot_pid) || !(globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN))
        return MDBX_BAD_RSLOT;
      thread_rthc_set(env->me_txkey, nullptr);
    }
  } else {
    eASSERT0(env, (env->flags & MDBX_NOSTICKYTHREADS));
  }

  bsr_t brs = mvcc_bind_slot(env);
  if (likely(brs.err == MDBX_SUCCESS)) {
    tASSERT1(txn, ro_slot_pid(brs.slot) == osal_getpid());
    tASSERT1(txn, ro_slot_tid(brs.slot) == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()));
  }

  txn->ro.slot = brs.slot;
  return brs.err;
}

static void ro_slot_release(MDBX_txn *txn) {
  tASSERT0(txn, is_ro_txn(txn) && txn->ro.slot);
  reader_slot_t *slot = txn->ro.slot;
  MDBX_env *const env = txn->env;
  eASSERT0(env, ro_slot_pid(slot) == osal_getpid());
  eASSERT0(env, slot->txnid.weak >= SAFE64_INVALID_THRESHOLD);
  if (ro_slot_pid(slot) == osal_getpid()) {
    safe64_reset(&slot->txnid, false);
    if ((env->flags & ENV_TXKEY) == 0)
      atomic_store32(&slot->pid, 0, mo_Relaxed);
  }
  txn->ro.slot = nullptr;
}

static inline int ro_slot_get(MDBX_txn *txn) {
  reader_slot_t *slot = txn->ro.slot;
  STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(slot->tid));
  if (likely(slot))
    return ro_slot_checkpid(txn);
  return ro_slot_alloc(txn);
}

static int ro_slot_clean(MDBX_txn *txn) {
  reader_slot_t *const slot = txn->ro.slot;
  if (unlikely(!slot))
    return MDBX_SUCCESS;

  MDBX_env *const env = txn->env;
  if (unlikely(!env->lck)) {
    txn->ro.slot = nullptr;
    return MDBX_SUCCESS;
  }

  int rc = ro_slot_checkpid(txn);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  if (likely((txn->flags & MDBX_TXN_FINISHED) == 0)) {
    if (likely((txn->flags & MDBX_TXN_PARKED) == 0)) {
      ENSURE_OBJ(env, txn->txnid >= /* paranoia is appropriate here */ env->lck->cached_oldest_txnid.weak);
      eASSERT0(env, txn->txnid == slot->txnid.weak && slot->txnid.weak >= env->lck->cached_oldest_txnid.weak);
    } else {
      txn->flags -= MDBX_TXN_PARKED;
      if (safe64_read(&slot->tid) == MDBX_TID_TXN_OUSTED)
        txn->flags |= MDBX_TXN_OUSTED;
      do {
        safe64_reset(&slot->txnid, false);
        atomic_store64(&slot->tid, txn->owner, mo_AcquireRelease);
        atomic_yield();
      } while (unlikely(safe64_read(&slot->txnid) < SAFE64_INVALID_THRESHOLD || safe64_read(&slot->tid) != txn->owner));
    }
    dxb_sanitize_tail(env, nullptr);
    atomic_store32(&slot->snapshot_pages_used, 0, mo_Relaxed);
    safe64_reset(&slot->txnid, true);
    atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed);
  } else {
    ENSURE_OBJ(env, safe64_read(&slot->txnid) >= /* paranoia is appropriate here */ SAFE64_INVALID_THRESHOLD);
  }
  return MDBX_SUCCESS;
}

static int ro_seize(MDBX_txn *txn) {
  /* Seek & fetch the last meta */
  troika_t troika = meta_tap(txn->env);
  uint64_t timestamp = 0;
  size_t loop = 0;
  txn->flags &= ~MDBX_TXN_FINISHED;
  do {
    MDBX_env *const env = txn->env;
    const meta_ptr_t head = likely(env->stuck_meta < 0) ? /* regular */ meta_recent(env, &troika)
                                                        : /* recovery mode */ meta_ptr(env, env->stuck_meta);
    reader_slot_t *const slot = txn->ro.slot;
    if (likely(slot != nullptr)) {
      safe64_reset(&slot->txnid, true);
      atomic_store32(&slot->snapshot_pages_used, head.ptr_v->geometry.first_unallocated, mo_Relaxed);
      atomic_store64(&slot->snapshot_pages_retired, unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired),
                     mo_Relaxed);
      safe64_write(&slot->txnid, head.txnid);
      eASSERT1(env, ro_slot_pid(slot) == osal_getpid());
      eASSERT1(env, ro_slot_tid(slot) == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()));
      eASSERT0(env, ro_slot_txnid(slot) == head.txnid || (ro_slot_txnid(slot) >= SAFE64_INVALID_THRESHOLD &&
                                                          head.txnid < env->lck->cached_oldest_txnid.weak));
      atomic_store32(&env->lck->rdt_refresh_flag, true, mo_AcquireRelease);
    } else {
      /* exclusive mode without lck */
      eASSERT0(env, !env->lck_mmap.lck && env->lck == lckless_stub(env));
    }
    jitter4testing(true);

    if (unlikely(meta_should_retry(env, &troika))) {
      timestamp = 0;
      continue;
    }

    /* Snap the state from current meta-head */
    int err = coherency_fetch_head(txn, head, &timestamp);
    jitter4testing(false);
    if (unlikely(err != MDBX_SUCCESS)) {
      if (err != MDBX_RESULT_TRUE)
        return err;
      continue;
    }

    const uint64_t snap_oldest = atomic_load64(&env->lck->cached_oldest_txnid, mo_AcquireRelease);
    if (unlikely(txn->txnid < snap_oldest)) {
      if (env->stuck_meta >= 0) {
        ERROR("target meta-page %i is referenced to an obsolete MVCC-snapshot "
              "%" PRIaTXN " < cached-oldest %" PRIaTXN,
              env->stuck_meta, txn->txnid, snap_oldest);
        return MDBX_MVCC_RETARDED;
      }
      continue;
    }

    if (!slot || likely(txn->txnid == atomic_load64(&slot->txnid, mo_Relaxed)))
      return MDBX_SUCCESS;

  } while (likely(++loop < 42));

  ERROR("bailout waiting for valid snapshot (%s)", "meta-pages are too volatile");
  return MDBX_PROBLEM;
}

static int ro_start_continue(MDBX_txn *txn) {
  MDBX_env *const env = txn->env;
  reader_slot_t *slot = txn->ro.slot;

  txn->owner =
      likely(slot) ? (uintptr_t)ro_slot_tid(slot) : ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self());

  if ((env->flags & MDBX_NOSTICKYTHREADS) == 0 && env->txn && unlikely(env->basal_txn->owner == txn->owner) &&
      (globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0)
    return MDBX_TXN_OVERLAPPING;

  int err = ro_seize(txn);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  err = txn_setup_primal(txn);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  return MDBX_SUCCESS;
}

int txn_ro_start(MDBX_txn *txn, bool prepare_only) {
  MDBX_env *const env = txn->env;
  txn->flags = txn_ro_flat | MDBX_TXN_FINISHED | (env->flags & (MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS));
  int err = ro_slot_get(txn);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  if (prepare_only) {
    eASSERT0(env, txn->txnid == 0);
    eASSERT0(env, txn->owner == 0);
    eASSERT0(env, txn->n_dbi == 0);
    reader_slot_t *slot = txn->ro.slot;
    if (likely(slot)) {
      eASSERT0(env, slot->snapshot_pages_used.weak == 0);
      eASSERT0(env, slot->txnid.weak >= SAFE64_INVALID_THRESHOLD);
      atomic_store32(&slot->snapshot_pages_used, 0, mo_Relaxed);
    }
    txn->flags = txn_ro_flat | MDBX_TXN_FINISHED;
    return MDBX_SUCCESS;
  }

  err = ro_start_continue(txn);
  if (unlikely(err != MDBX_SUCCESS)) {
    txn->flags |= /* to avoid ENSURE-fail inside ro_slot_clean() */ MDBX_TXN_PARKED;
    txn_ro_reset(txn);
    return err;
  }

  eASSERT0(env, pgno2bytes(env, txn->geo.first_unallocated) <= env->dxb_mmap.current);
  eASSERT0(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
#if defined(_WIN32) || defined(_WIN64)
  const size_t used_bytes = pgno2bytes(env, txn->geo.first_unallocated);
  if (((used_bytes > env->geo_in_bytes.lower && env->geo_in_bytes.shrink) ||
       (globals.running_under_Wine &&
        /* under Wine acquisition of remap_lock is always required,
         * since Wine don't support section extending,
         * i.e. in both cases unmap+map are required. */
        used_bytes < env->geo_in_bytes.upper && env->geo_in_bytes.grow)) &&
      /* avoid recursive use SRW */ (txn->flags & MDBX_NOSTICKYTHREADS) == 0) {
    imports.srwl_AcquireShared(&env->remap_lock);
    txn->flags |= txn_shrink_allowed;
  }
#endif /* Windows */

  dxb_sanitize_tail(env, txn);
  ENSURE_OBJ(env, txn->txnid >=
                      /* paranoia is appropriate here */ env->lck->cached_oldest_txnid.weak);
  cASSERT0(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY);
  cASSERT0(txn, check_table_flags(txn->dbs[MAIN_DBI].flags));
  return MDBX_SUCCESS;
}

int txn_ro_reset(MDBX_txn *txn) {
  DEBUG("%s txn %" PRIaTXN "%c-0x%X %p  on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, "reset", txn->txnid, 'r',
        txn->flags, __Wpedantic_format_voidptr(txn), __Wpedantic_format_voidptr(txn->env), txn->dbs[MAIN_DBI].root,
        txn->dbs[FREE_DBI].root);

  cASSERT0(txn, is_ro_txn(txn));
  if (txn->flags & txn_may_have_cursors)
    txn_done_cursors(txn);

  cASSERT0(txn, (txn->flags & txn_may_have_cursors) == 0);
  txn->n_dbi = 0; /* prevent further DBI activity */

#if defined(_WIN32) || defined(_WIN64)
  if ((txn->flags & (MDBX_TXN_FINISHED | txn_shrink_allowed)) == txn_shrink_allowed)
    imports.srwl_ReleaseShared(&txn->env->remap_lock);
#endif /* Windows */

  int rc = ro_slot_clean(txn);
  txn->flags |= MDBX_TXN_FINISHED;
  txn->owner = 0;
  txn->txnid = INVALID_TXNID;
  return rc;
}

void txn_ro_free(MDBX_txn *txn) {
  cASSERT0(txn, is_ro_txn(txn));
  if (txn->ro.slot) {
    txn_ro_reset(txn);
    ro_slot_release(txn);
  }

  cASSERT0(txn, (txn->flags & txn_may_have_cursors) == 0);
  txn->signature = 0;
  osal_free(txn);
}

int txn_ro_park(MDBX_txn *txn, bool autounpark) {
  reader_slot_t *const slot = txn->ro.slot;
  cASSERT0(txn, (txn->flags & (MDBX_TXN_FINISHED | txn_ro_flat | MDBX_TXN_PARKED)) == txn_ro_flat);
  cASSERT0(txn, ro_slot_tid(slot) < MDBX_TID_TXN_OUSTED);
  if (unlikely((txn->flags & (MDBX_TXN_FINISHED | txn_ro_flat | MDBX_TXN_PARKED)) != txn_ro_flat))
    return MDBX_BAD_TXN;

  const mdbx_pid_t pid = atomic_load_pid(&slot->pid, mo_Relaxed);
  const uint64_t tid = atomic_load64(&slot->tid, mo_Relaxed);
  const txnid_t txnid = atomic_load64(&slot->txnid, mo_Relaxed);
  if (unlikely(pid != txn->env->pid)) {
    ERROR("unexpected pid %zu%s%zu", (size_t)pid, " != must ", (size_t)txn->env->pid);
    return MDBX_PROBLEM;
  }
  if (unlikely(tid != txn->owner || txnid != txn->txnid)) {
    ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%0zx"
          " and/or txn-id %" PRIaTXN "%s%" PRIaTXN,
          tid, " != must ", txn->owner, txnid, " != must ", txn->txnid);
    return MDBX_BAD_RSLOT;
  }

  if (unlikely((txn->flags & MDBX_TXN_ERROR)))
    return ro_slot_clean(txn);

  atomic_store64(&slot->tid, MDBX_TID_TXN_PARKED, mo_AcquireRelease);
  atomic_store32(&txn->env->lck->rdt_refresh_flag, true, mo_Relaxed);
  txn->flags += autounpark ? MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK : MDBX_TXN_PARKED;
  return MDBX_SUCCESS;
}

int txn_ro_unpark(MDBX_txn *txn) {
  if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD | txn_ro_flat | MDBX_TXN_PARKED)) !=
               (txn_ro_flat | MDBX_TXN_PARKED)))
    return (txn->flags & MDBX_TXN_OUSTED) ? MDBX_OUSTED : MDBX_BAD_TXN;

  for (reader_slot_t *const slot = txn->ro.slot; slot; atomic_yield()) {
    const mdbx_pid_t pid = atomic_load_pid(&slot->pid, mo_Relaxed);
    uint64_t tid = safe64_read(&slot->tid);
    uint64_t txnid = safe64_read(&slot->txnid);
    if (unlikely(pid != txn->env->pid)) {
      ERROR("unexpected pid %zu%s%zu", (size_t)pid, " != expected ", (size_t)txn->env->pid);
      return MDBX_PROBLEM;
    }
    if (unlikely(tid == MDBX_TID_TXN_OUSTED || txnid >= SAFE64_INVALID_THRESHOLD))
      break;
    if (unlikely(tid != MDBX_TID_TXN_PARKED || txnid != txn->txnid)) {
      ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%" PRIx64 " and/or txn-id %" PRIaTXN "%s%" PRIaTXN, tid, " != must ",
            MDBX_TID_TXN_OUSTED, txnid, " != must ", txn->txnid);
      break;
    }
    if (unlikely((txn->flags & MDBX_TXN_ERROR)))
      break;

#if MDBX_64BIT_CAS
    if (unlikely(!atomic_cas64(&slot->tid, MDBX_TID_TXN_PARKED, txn->owner)))
      continue;
#else
    atomic_store32(&slot->tid.high, (uint32_t)((uint64_t)txn->owner >> 32), mo_Relaxed);
    if (unlikely(!atomic_cas32(&slot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)txn->owner))) {
      atomic_store32(&slot->tid.high, (uint32_t)(MDBX_TID_TXN_PARKED >> 32), mo_AcquireRelease);
      continue;
    }
#endif
    txnid = safe64_read(&slot->txnid);
    tid = safe64_read(&slot->tid);
    if (unlikely(txnid != txn->txnid || tid != txn->owner)) {
      ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%zx"
            " and/or txn-id %" PRIaTXN "%s%" PRIaTXN,
            tid, " != must ", txn->owner, txnid, " != must ", txn->txnid);
      break;
    }
    txn->flags &= ~(MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK);
    return MDBX_SUCCESS;
  }

  int err = txn_ro_reset(txn);
  return (err == MDBX_SUCCESS && (txn->flags & MDBX_TXN_OUSTED)) ? MDBX_OUSTED : err;
}

int txn_ro_clone(const MDBX_txn *const origin, MDBX_txn *const clone) {
  int err = ro_slot_get(clone);
  if (unlikely(err != MDBX_SUCCESS))
    return err;

  if (unlikely((origin->flags & MDBX_TXN_FINISHED) || origin->txnid < MIN_TXNID || origin->txnid > MAX_TXNID))
    return MDBX_BAD_TXN;

  clone->flags = (origin->flags & (MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | MDBX_TXN_AUTOUNPARK | MDBX_TXN_PARKED)) |
                 txn_ro_flat | MDBX_TXN_FINISHED;
  clone->owner = likely(clone->ro.slot) ? (uintptr_t)ro_slot_tid(clone->ro.slot)
                                        : ((clone->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self());
  clone->txnid = txn_basis_snapshot(origin);
  clone->front_txnid = clone->txnid;

  MDBX_env *const env = origin->env;
  if (origin->flags & txn_ro_flat) {
    if ((clone->flags & MDBX_NOSTICKYTHREADS) == 0 && env->txn && unlikely(env->basal_txn->owner == clone->owner) &&
        (globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0)
      return MDBX_TXN_OVERLAPPING;

    if (likely(clone->ro.slot)) {
      const uint32_t pages_used = origin->ro.slot ? origin->ro.slot->snapshot_pages_used.weak : 0;
      const uint64_t pages_retired = origin->ro.slot ? origin->ro.slot->snapshot_pages_retired.weak : 0;
      atomic_store32(&clone->ro.slot->snapshot_pages_used, pages_used, mo_Relaxed);
      atomic_store64(&clone->ro.slot->snapshot_pages_retired, pages_retired, mo_Relaxed);
      safe64_write(&clone->ro.slot->txnid, (clone->flags & MDBX_TXN_PARKED) ? MDBX_TID_TXN_PARKED : clone->txnid);
    } else {
      /* exclusive mode without lck */
      eASSERT0(env, !env->lck_mmap.lck && env->lck == lckless_stub(env));
    }

    /* Setup db info */
    clone->geo = origin->geo;
    clone->canary = origin->canary;
    memcpy(clone->dbs, origin->dbs, sizeof(clone->dbs[0]) * CORE_DBS);
    cASSERT0(clone, clone->dbs[FREE_DBI].flags == MDBX_INTEGERKEY);
    cASSERT0(clone, check_table_flags(clone->dbs[MAIN_DBI].flags));
    VALGRIND_MAKE_MEM_UNDEFINED(clone->dbi_state, env->max_dbi);
#if MDBX_ENABLE_DBI_SPARSE
    clone->n_dbi = CORE_DBS;
    VALGRIND_MAKE_MEM_UNDEFINED(clone->dbi_sparse,
                                ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(clone->dbi_sparse[0])) / CHAR_BIT);
    clone->dbi_sparse[0] = (1u << CORE_DBS) - 1;
#else
    clone->n_dbi = (env->n_dbi < 8) ? env->n_dbi : 8;
    if (clone->n_dbi > CORE_DBS)
      memset(clone->dbi_state + CORE_DBS, 0, clone->n_dbi - CORE_DBS);
#endif /* MDBX_ENABLE_DBI_SPARSE */
    clone->dbi_state[FREE_DBI] = DBI_LINDO | DBI_VALID;
    clone->dbi_state[MAIN_DBI] = DBI_LINDO | DBI_VALID;
    clone->cursors[FREE_DBI] = nullptr;
    clone->cursors[MAIN_DBI] = nullptr;
    clone->dbi_seqs[FREE_DBI] = origin->dbi_seqs[FREE_DBI];
    clone->dbi_seqs[MAIN_DBI] = origin->dbi_seqs[MAIN_DBI];

    if (!(clone->flags & MDBX_TXN_PARKED)) {
      if (likely(clone->ro.slot) && unlikely(clone->txnid != atomic_load64(&clone->ro.slot->txnid, mo_Relaxed)))
        return MDBX_OUSTED;
      if (unlikely(clone->txnid < atomic_load64(&env->lck->cached_oldest_txnid, mo_AcquireRelease)))
        return MDBX_MVCC_RETARDED;
    }
    return MDBX_SUCCESS;
  }

  return ro_seize(clone);
}

pgop_stat_t *txn_latency_gcprof(const MDBX_env *env, MDBX_commit_latency *latency) {
  pgop_stat_t *const pgops = &env->lck->pgops;
  latency->gc_prof.work_counter = pgops->gc_prof.work.spe_counter;
  latency->gc_prof.work_rtime_monotonic = osal_monotime_to_16dot16(pgops->gc_prof.work.rtime_monotonic);
  latency->gc_prof.work_xtime_cpu = osal_monotime_to_16dot16(pgops->gc_prof.work.xtime_cpu);
  latency->gc_prof.work_rsteps = pgops->gc_prof.work.rsteps;
  latency->gc_prof.work_xpages = pgops->gc_prof.work.xpages;
  latency->gc_prof.work_majflt = pgops->gc_prof.work.majflt;

  latency->gc_prof.self_counter = pgops->gc_prof.self.spe_counter;
  latency->gc_prof.self_rtime_monotonic = osal_monotime_to_16dot16(pgops->gc_prof.self.rtime_monotonic);
  latency->gc_prof.self_xtime_cpu = osal_monotime_to_16dot16(pgops->gc_prof.self.xtime_cpu);
  latency->gc_prof.self_rsteps = pgops->gc_prof.self.rsteps;
  latency->gc_prof.self_xpages = pgops->gc_prof.self.xpages;
  latency->gc_prof.self_majflt = pgops->gc_prof.self.majflt;

  latency->gc_prof.wloops = pgops->gc_prof.wloops;
  latency->gc_prof.coalescences = pgops->gc_prof.coalescences;
  latency->gc_prof.wipes = pgops->gc_prof.wipes;
  latency->gc_prof.flushes = pgops->gc_prof.flushes;
  latency->gc_prof.kicks = pgops->gc_prof.kicks;

  latency->gc_prof.pnl_merge_work.time = osal_monotime_to_16dot16(pgops->gc_prof.work.pnl_merge.time);
  latency->gc_prof.pnl_merge_work.calls = pgops->gc_prof.work.pnl_merge.calls;
  latency->gc_prof.pnl_merge_work.volume = pgops->gc_prof.work.pnl_merge.volume;
  latency->gc_prof.pnl_merge_self.time = osal_monotime_to_16dot16(pgops->gc_prof.self.pnl_merge.time);
  latency->gc_prof.pnl_merge_self.calls = pgops->gc_prof.self.pnl_merge.calls;
  latency->gc_prof.pnl_merge_self.volume = pgops->gc_prof.self.pnl_merge.volume;

  latency->gc_prof.max_reader_lag = pgops->gc_prof.max_reader_lag;
  latency->gc_prof.max_retained_pages = pgops->gc_prof.max_retained_pages;
  return pgops;
}

__hot bool txn_gc_detent(const MDBX_txn *const txn) {
  const txnid_t detent = mvcc_shapshot_oldest_rw(txn).oldest_txnid;
  if (likely(detent == txn->env->gc.detent))
    return false;

  txn->env->gc.detent = detent;
  return true;
}

void txn_done_cursors(MDBX_txn *txn) {
  cASSERT0(txn, txn->flags & txn_may_have_cursors);

  TXN_FOREACH_DBI_ALL(txn, i) {
    MDBX_cursor *cursor = txn->cursors[i];
    if (cursor) {
      txn->cursors[i] = nullptr;
      do
        cursor = cursor_eot(cursor, txn);
      while (cursor);
    }
  }
  txn->flags &= ~txn_may_have_cursors;
}

int txn_shadow_cursors(const MDBX_txn *parent, const size_t dbi) {
  cASSERT0(parent, dbi < parent->n_dbi);
  MDBX_cursor *cursor = parent->cursors[dbi];
  if (!cursor)
    return MDBX_SUCCESS;

  MDBX_txn *const txn = parent->nested;
  cASSERT0(parent, parent->flags & txn_may_have_cursors);
  MDBX_cursor *next = nullptr;
  do {
    next = cursor->next;
    if (cursor->signature != cur_signature_live) {
      ENSURE_OBJ(parent, cursor->signature == cur_signature_wait4eot);
      continue;
    }
    cASSERT0(parent, cursor->txn == parent && dbi == cursor_dbi(cursor));

    int err = cursor_shadow(cursor, txn, dbi);
    if (unlikely(err != MDBX_SUCCESS)) {
      /* не получилось забекапить курсоры */
      txn->dbi_state[dbi] = DBI_OLDEN | DBI_LINDO;
      txn->flags |= MDBX_TXN_ERROR;
      return err;
    }
    cursor->next = txn->cursors[dbi];
    txn->cursors[dbi] = cursor;
    txn->flags |= txn_may_have_cursors;
  } while ((cursor = next) != nullptr);
  return MDBX_SUCCESS;
}

int txn_commit(MDBX_txn *txn, MDBX_commit_latency *latency, struct commit_timestamp *ts) {
  cASSERT0(txn, !txn->nested && !(txn->flags & MDBX_TXN_FINISHED));
  if (unlikely(txn->flags & MDBX_TXN_ERROR)) {
    int err = txn_abort(txn, latency);
    return (err == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : err;
  }

  if (MDBX_ENABLE_FAKE_NESTED_READONLY_TRANSACTIONS && unlikely(txn->flags & txn_ro_nested))
    return txn_nested_fakero_end(txn);

  MDBX_env *const env = txn->env;
  if (txn != env->txn) {
    if (unlikely(txn->flags & txn_ro_flat) == 0) {
      ERROR("attempt to commit %s txn %p", "unknown", __Wpedantic_format_voidptr(txn));
      return MDBX_EINVAL;
    }
    if (unlikely(txn->parent || (txn->flags & MDBX_TXN_HAS_CHILD) || txn == env->basal_txn)) {
      ERROR("attempt to commit %s txn %p", "strange read-only", __Wpedantic_format_voidptr(txn));
      return MDBX_PROBLEM;
    }
    txn_ro_free(txn);
    return MDBX_SUCCESS;
  }

  if (txn == env->basal_txn) {
    int rc = txn_basal_commit(txn, ts);
    if (unlikely(rc != MDBX_SUCCESS)) {
      txn->flags |= MDBX_TXN_ERROR;
      if (rc == MDBX_RESULT_TRUE)
        rc = MDBX_NOSUCCESS_PURE_COMMIT ? MDBX_RESULT_TRUE : MDBX_SUCCESS;
    }
    if (latency) {
      pgop_stat_t *const pgops = txn_latency_gcprof(env, latency);
      memset(&pgops->gc_prof, 0, sizeof(pgops->gc_prof));
    }
    int err = txn_basal_end(txn, true);
    return (err == MDBX_SUCCESS) ? rc : err;
  }

  if (unlikely(!txn->parent || txn->parent->nested != txn || txn->parent->env != env)) {
    ERROR("attempt to commit %s txn %p", "strange nested", __Wpedantic_format_voidptr(txn));
    return MDBX_PROBLEM;
  }
  int rc = txn_nested_commit(txn, ts);
  if (latency)
    txn_latency_gcprof(env, latency);
  return rc;
}

#if !(defined(_WIN32) || defined(_WIN64))
void txn_abort_after_resurrect(MDBX_txn *txn) {
  if (likely(txn->signature == txn_signature)) {
    if (txn->nested) {
      txn_abort_after_resurrect(txn->nested);
      cASSERT0(txn, !txn->nested);
    }
    txn_abort(txn, nullptr);
  }
}
#endif /* Windows */

int txn_abort(MDBX_txn *txn, MDBX_commit_latency *latency) {
  DEBUG("txn %" PRIaTXN "%c-0x%X %p  on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid,
        (txn->flags & txn_ro_flat) ? 'r' : 'w', txn->flags, __Wpedantic_format_voidptr(txn),
        __Wpedantic_format_voidptr(txn->env), txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root);

  cASSERT0(txn, txn->signature == txn_signature && !txn->nested && !(txn->flags & MDBX_TXN_HAS_CHILD));
  tASSERT1(txn, (txn->flags & (MDBX_TXN_ERROR | txn_ro_flat)) || txn_dpl_check(txn));

  if (latency) {
    txn_latency_gcprof(txn->env, latency);
    if (txn != txn->env->txn) {
      MDBX_commit_latency snap;
      do {
        snap = *latency;
        txn_latency_gcprof(txn->env, latency);
      } while (unlikely(memcpy(latency, &snap, sizeof(snap))));
    }
  }

  if (MDBX_ENABLE_FAKE_NESTED_READONLY_TRANSACTIONS && unlikely(txn->flags & txn_ro_nested))
    return txn_nested_fakero_end(txn);

  txn->flags |= /* avoid merge cursors' state */ MDBX_TXN_ERROR;

  if (txn->flags & txn_may_have_cursors)
    txn_done_cursors(txn);

  MDBX_env *const env = txn->env;
  MDBX_txn *const parent = txn->parent;
  if (txn == env->basal_txn) {
    cASSERT0(txn, !parent);
    if (unlikely(!txn->owner))
      return MDBX_BAD_TXN;
    return txn_basal_end(txn, true);
  }

  if (txn->parent)
    return txn_nested_abort(txn);

  cASSERT0(txn, (txn->flags & (txn_ro_flat | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD)) == txn_ro_flat);
  txn_ro_free(txn);
  return MDBX_SUCCESS;
}

typedef struct seq_latch_result {
  int err;
  uint32_t seq;
} slr_t;

__cold static slr_t latch_maindb_locked(MDBX_txn *txn, MDBX_env *const env) {
  slr_t slr = {.err = MDBX_SUCCESS, .seq = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease)};
  if (env->dbs_flags[MAIN_DBI] & DB_VALID /* Флаги MainDB уже были загружены? */) {
    /* Читаем и проверяем повторно после захвата dbi-блокировки */
    if (env->dbs_flags[MAIN_DBI] == (DB_VALID | txn->dbs[MAIN_DBI].flags))
      return slr;
    if (env->basal_txn && env->txn != txn) {
      /* Параллельно выполняется пишущая транзакция, которая вероятно и произвела изменение MainDB.
       * Как-либо отказаться от использования новых атрибутов в рамках текущего процесса невозможно. */
      ERROR("MainDB db-flags changes 0x%x -> 0x%x ahead of read-txn %" PRIaTXN, txn->dbs[MAIN_DBI].flags,
            env->dbs_flags[MAIN_DBI] & ~DB_VALID, txn->txnid);
      slr.err = MDBX_INCOMPATIBLE;
      return slr;
    }

    NOTICE("renew MainDB for %s-txn %" PRIaTXN " since db-flags changes 0x%x -> 0x%x",
           (txn->flags & txn_ro_flat) ? "ro" : "rw", txn->txnid, env->dbs_flags[MAIN_DBI] & ~DB_VALID,
           txn->dbs[MAIN_DBI].flags);
    slr.seq = dbi_seq_next(env, MAIN_DBI);
    env->dbs_flags[MAIN_DBI] = DB_POISON;
    atomic_store32(&env->dbi_seqs[MAIN_DBI], slr.seq, mo_AcquireRelease);
  }

  slr.err = tbl_setup(env, &env->kvs[MAIN_DBI], &txn->dbs[MAIN_DBI]);
  if (likely(slr.seq == MDBX_SUCCESS)) {
    slr.seq = dbi_seq_next(env, MAIN_DBI);
    env->dbs_flags[MAIN_DBI] = DB_VALID | txn->dbs[MAIN_DBI].flags;
    atomic_store32(&env->dbi_seqs[MAIN_DBI], slr.seq, mo_AcquireRelease);
  }
  return slr;
}

int txn_setup_primal(MDBX_txn *txn) {
  MDBX_env *const env = txn->env;

  if (unlikely(txn->txnid < MIN_TXNID || txn->txnid > MAX_TXNID)) {
    ERROR("%s", "environment corrupted by died writer, must shutdown!");
    return MDBX_CORRUPTED;
  }
  txn->front_txnid = txn->txnid + ((txn->flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == 0);

  /* Setup db info */
  cASSERT0(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY);
  cASSERT0(txn, check_table_flags(txn->dbs[MAIN_DBI].flags));
  VALGRIND_MAKE_MEM_UNDEFINED(txn->dbi_state, env->max_dbi);
#if MDBX_ENABLE_DBI_SPARSE
  txn->n_dbi = CORE_DBS;
  VALGRIND_MAKE_MEM_UNDEFINED(txn->dbi_sparse,
                              ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT);
  txn->dbi_sparse[0] = (1u << CORE_DBS) - 1;
#else
  txn->n_dbi = (env->n_dbi < 8) ? env->n_dbi : 8;
  if (txn->n_dbi > CORE_DBS)
    memset(txn->dbi_state + CORE_DBS, 0, txn->n_dbi - CORE_DBS);
#endif /* MDBX_ENABLE_DBI_SPARSE */
  txn->dbi_state[FREE_DBI] = DBI_LINDO | DBI_VALID;
  txn->dbi_state[MAIN_DBI] = DBI_LINDO | DBI_VALID;
  txn->cursors[FREE_DBI] = nullptr;
  txn->cursors[MAIN_DBI] = nullptr;
  txn->dbi_seqs[FREE_DBI] = 0;
  txn->dbi_seqs[MAIN_DBI] = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease);

  if (unlikely(env->dbs_flags[MAIN_DBI] != (DB_VALID | txn->dbs[MAIN_DBI].flags) || !txn->dbi_seqs[MAIN_DBI])) {
    int err = osal_fastmutex_acquire(&env->dbi_lock);
    if (unlikely(err != MDBX_SUCCESS)) {
      ERROR("dbi_lock failed, err %d", err);
      return err;
    }

    slr_t slr = latch_maindb_locked(txn, env);
    ENSURE_OBJ(env, osal_fastmutex_release(&env->dbi_lock) == MDBX_SUCCESS);
    if (unlikely((err = slr.err) != MDBX_SUCCESS))
      return err;
    txn->dbi_seqs[MAIN_DBI] = slr.seq;
  }
  cASSERT0(txn, check_table_flags(txn->dbs[MAIN_DBI].flags) && txn->dbi_seqs[MAIN_DBI]);

  if (unlikely(txn->dbs[FREE_DBI].flags != MDBX_INTEGERKEY)) {
    ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, "GC/FreeDB");
    return MDBX_INCOMPATIBLE;
  }

  if (unlikely(env->flags & ENV_FATAL_ERROR)) {
    WARNING("%s", "environment had fatal error, must shutdown!");
    return MDBX_PANIC;
  }

  const size_t size_bytes = pgno2bytes(env, txn->geo.end_pgno);
  const size_t used_bytes = pgno2bytes(env, txn->geo.first_unallocated);
  const size_t required_bytes = (txn->flags & txn_ro_flat) ? used_bytes : size_bytes;
  eASSERT0(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
  int err = MDBX_SUCCESS;
  if (unlikely(required_bytes > env->dxb_mmap.current)) {
    /* Размер БД (для пишущих транзакций) или используемых данных (для
     * читающих транзакций) больше предыдущего/текущего размера внутри
     * процесса, увеличиваем. Сюда также попадает случай увеличения верхней
     * границы размера БД и отображения. */
    if (txn->geo.upper > MAX_PAGENO + 1 || bytes2pgno(env, pgno2bytes(env, txn->geo.upper)) != txn->geo.upper)
      return MDBX_UNABLE_EXTEND_MAPSIZE;
    err = dxb_resize(env, txn->geo.first_unallocated, txn->geo.end_pgno, txn->geo.upper, implicit_grow);
    eASSERT0(env, err != MDBX_SUCCESS || env->dxb_mmap.limit >= env->dxb_mmap.current);
    return err;
  }

  /* LY: Проверка условия size_bytes < env->dxb_mmap.current проверяется до захвата блокировки, так как при существующей
   * текущей транзакции допустимое конкурирующее изменение env->dxb_mmap.current в другом потоке не повлияет на
   * дальнейшую логику. Иначе говоря, такая проверка до захвата блокировки полностью безопасна и не может создавать
   * проблем, но при этом избавляет от затрат на захват и освобождение блокировки. */
  /* coverity[LOCK_EVASION] */
  if (unlikely(size_bytes < env->dxb_mmap.current)) {
    /* Размер БД меньше предыдущего/текущего размера внутри процесса, можно
     * уменьшить, но всё сложнее:
     *  - размер файла ДОЛЖЕН бать согласован со всеми читаемыми снимками
     *    на момент коммита последней транзакции;
     *  - в читающей транзакции размер файла может быть больше и него нельзя
     *    изменять, в том числе менять madvise (меньша размера файла нельзя,
     *    а за размером нет смысла).
     *  - в пишущей транзакции уменьшать размер файла можно только после
     *    проверки размера читаемых снимков, но в этом нет смысла, так как
     *    это будет сделано при фиксации транзакции.
     *
     *  В сухом остатке, можно только установить dxb_mmap.current равным
     *  размеру файла, а это проще сделать без вызова dxb_resize() и усложения
     *  внутренней логики.
     *
     *  В этой тактике есть недостаток: если пишущите транзакции не регулярны,
     *  и при завершении такой транзакции файл БД остаётся не-уменьшеным из-за
     *  читающих транзакций использующих предыдущие снимки. */
#if defined(_WIN32) || defined(_WIN64)
    imports.srwl_AcquireShared(&env->remap_lock);
#else
    err = osal_fastmutex_acquire(&env->remap_lock);
    if (unlikely(err != MDBX_SUCCESS)) {
      ERROR("remap_lock failed, err %d", err);
      return err;
    }
#endif
    eASSERT0(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
    err = osal_filesize(env->dxb_mmap.fd, &env->dxb_mmap.filesize);
    if (likely(err == MDBX_SUCCESS)) {
      eASSERT0(env, env->dxb_mmap.filesize >= required_bytes);
      if (env->dxb_mmap.current > env->dxb_mmap.filesize)
        env->dxb_mmap.current =
            (env->dxb_mmap.limit < env->dxb_mmap.filesize) ? env->dxb_mmap.limit : (size_t)env->dxb_mmap.filesize;
    }
#if defined(_WIN32) || defined(_WIN64)
    imports.srwl_ReleaseShared(&env->remap_lock);
#else
    ENSURE_OBJ(env, osal_fastmutex_release(&env->remap_lock) == MDBX_SUCCESS);
#endif
  }

  return err;
}

int txn_check_badbits_parked(const MDBX_txn *txn, int bad_bits) {
  cASSERT0(txn, (bad_bits & MDBX_TXN_PARKED) && (txn->flags & bad_bits));
  /* Здесь осознано заложено отличие в поведении припаркованных транзакций:
   *  - некоторые функции (например mdbx_env_info_ex()), допускают
   *    использование поломанных транзакций (с флагом MDBX_TXN_ERROR), но
   *    не могут работать с припаркованными транзакциями (требуют распарковки).
   *  - но при распарковке поломанные транзакции завершаются.
   *  - получается что транзакцию можно припарковать, потом поломать вызвав
   *    mdbx_txn_break(), но далее любое её использование приведет к завершению
   *    при распарковке.
   *
   * Поэтому для припаркованных транзакций возвращается ошибка если не-включена
   * авто-распарковка, либо есть другие плохие биты. */
  if ((txn->flags & (bad_bits | MDBX_TXN_AUTOUNPARK)) != (MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK))
    return LOG_IFERR(MDBX_BAD_TXN);

  cASSERT0(txn, bad_bits == MDBX_TXN_BLOCKED || bad_bits == MDBX_TXN_BLOCKED - MDBX_TXN_ERROR);
  return mdbx_txn_unpark((MDBX_txn *)txn, false);
}

MDBX_txn *txn_alloc(const unsigned flags, MDBX_env *env) {
  MDBX_txn *txn = nullptr;
  const intptr_t bitmap_bytes =
#if MDBX_ENABLE_DBI_SPARSE
      ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT;
#else
      0;
#endif /* MDBX_ENABLE_DBI_SPARSE */
  STATIC_ASSERT(sizeof(txn->wr) > sizeof(txn->ro));
  const size_t base = (flags & txn_ro_flat) ? sizeof(MDBX_txn) - sizeof(txn->wr) + sizeof(txn->ro) : sizeof(MDBX_txn);
  const size_t size = base +
                      ((flags & txn_ro_flat) ? (size_t)bitmap_bytes + env->max_dbi * sizeof(txn->dbi_seqs[0]) : 0) +
                      env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_state[0]));
  txn = osal_malloc(size);
  if (unlikely(!txn))
    return txn;
  MDBX_ANALYSIS_ASSUME(size > base);
  memset(txn, 0, (MDBX_GOOFY_MSVC_STATIC_ANALYZER && base > size) ? size : base);

  txn->dbs = ptr_disp(txn, base);
  txn->cursors = ptr_disp(txn->dbs, env->max_dbi * sizeof(txn->dbs[0]));
#if MDBX_CHECKING > 0
  txn->cursors[FREE_DBI] = nullptr; /* avoid SIGSEGV in an assertion later */
#endif
  txn->dbi_state = ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0]));
  txn->flags = flags;
  txn->env = env;

  if (flags & txn_ro_flat) {
    txn->dbi_seqs = ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0]));
#if MDBX_ENABLE_DBI_SPARSE
    txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes);
#endif /* MDBX_ENABLE_DBI_SPARSE */
  }
  return txn;
}

MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED unsigned ceil_log2n(size_t value_uintptr) {
  ASSERT(value_uintptr > 0 && value_uintptr < INT32_MAX);
  value_uintptr -= 1;
  value_uintptr |= value_uintptr >> 1;
  value_uintptr |= value_uintptr >> 2;
  value_uintptr |= value_uintptr >> 4;
  value_uintptr |= value_uintptr >> 8;
  value_uintptr |= value_uintptr >> 16;
  return log2n_powerof2(value_uintptr + 1);
}

MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION unsigned log2n_powerof2(size_t value_uintptr) {
  ASSERT(value_uintptr > 0 && value_uintptr < INT32_MAX && is_powerof2(value_uintptr));
  ASSERT((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr);
  const uint32_t value_uint32 = (uint32_t)value_uintptr;
#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz)
  STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned));
  return __builtin_ctz(value_uint32);
#elif defined(_MSC_VER)
  unsigned long index;
  STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long));
  _BitScanForward(&index, value_uint32);
  return index;
#else
  static const uint8_t debruijn_ctz32[32] = {0,  1,  28, 2,  29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4,  8,
                                             31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6,  11, 5,  10, 9};
  return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27];
#endif
}

MDBX_NOTHROW_CONST_FUNCTION uint64_t rrxmrrxmsx_0(uint64_t v) {
  /* Pelle Evensen's mixer, https://bit.ly/2HOfynt */
  v ^= (v << 39 | v >> 25) ^ (v << 14 | v >> 50);
  v *= UINT64_C(0xA24BAED4963EE407);
  v ^= (v << 40 | v >> 24) ^ (v << 15 | v >> 49);
  v *= UINT64_C(0x9FB21C651E98DF25);
  return v ^ v >> 28;
}

__cold char *ratio2digits(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer, int precision) {
  ASSERT(d > 0 && precision < 20);
  char *const dot = buffer->string + 21;
  uint64_t i = v / d, f = v % d, m = d;

  char *tail = dot;
  bool carry = m - f < m / 2;
  if (precision > 0) {
    *tail = '.';
    do {
      while (unlikely(f > UINT64_MAX / 10)) {
        f >>= 1;
        m >>= 1;
      }
      f *= 10;
      ASSERT(tail > buffer->string && tail < ARRAY_END(buffer->string) - 1);
      *++tail = '0' + (char)(f / m);
      f %= m;
    } while (--precision && tail < ARRAY_END(buffer->string) - 1);

    carry = m - f < m / 2;
    for (char *scan = tail; carry && scan > dot; --scan)
      *scan = (carry = *scan == '9') ? '0' : *scan + 1;
  }
  ASSERT(tail > buffer->string && tail < ARRAY_END(buffer->string) - 1);
  *++tail = '\0';

  char *head = dot;
  i += carry;
  while (i > 9) {
    ASSERT(head > buffer->string && head < ARRAY_END(buffer->string));
    *--head = '0' + (char)(i % 10);
    i /= 10;
  }
  ASSERT(head > buffer->string && head < ARRAY_END(buffer->string));
  *--head = '0' + (char)i;

  return head;
}

__cold char *ratio2percent(uint64_t value, uint64_t whole, ratio2digits_buffer_t *buffer) {
  while (unlikely(value > UINT64_MAX / 100)) {
    value >>= 1;
    whole >>= 1;
  }
  const bool rough = whole >= value && (!value || value > whole / 16);
  return ratio2digits(value * 100, whole, buffer, rough ? 1 : 2);
}

MDBX_MAYBE_UNUSED bin128_t mul64x64_128_fallback(uint64_t x, uint64_t y) {
  bin128_t r;
#if MDBX_HAVE_NATIVE_U128 && MDBX_CHECKING < 1
  r.u128 = x;
  r.u128 *= y;
#else
  const uint64_t xl = x & UINT32_C(0xFFFFffff);
  const uint64_t xh = x >> 32;
  const uint64_t yl = y & UINT32_C(0xFFFFffff);
  const uint64_t yh = y >> 32;

  const uint64_t ll = xl * yl;
  const uint64_t hh = xh * yh;
  const uint64_t hl = xh * yl + (ll >> 32);
  const uint64_t lh = xl * yh + (hl & UINT32_C(0xFFFFffff));

  r.l = (lh << 32) | (ll & UINT32_C(0xFFFFffff));
  r.h = hh + (hl >> 32) + (lh >> 32);
#endif
  return r;
}

static page_type_t walk_page_type(const page_t *mp) {
  if (mp)
    switch (mp->flags & ~P_SPILLED) {
    case P_BRANCH:
      return page_branch;
    case P_LEAF:
      return page_leaf;
    case P_LEAF | P_DUPFIX:
      return page_dupfix_leaf;
    case P_LARGE:
      return page_large;
    }
  return page_broken;
}

static page_type_t walk_subpage_type(const page_t *sp) {
  switch (sp->flags & /* ignore legacy P_DIRTY flag */ ~P_LEGACY_DIRTY) {
  case P_LEAF | P_SUBP:
    return page_sub_leaf;
  case P_LEAF | P_DUPFIX | P_SUBP:
    return page_sub_dupfix_leaf;
  default:
    return page_sub_broken;
  }
}

/* Depth-first tree traversal. */
__cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, txnid_t parent_txnid,
                            const pgno_t parent_pgno) {
  ASSERT(pgno != P_INVALID);
  page_t *mp = nullptr;
  int err = page_get(ctx->cursor, pgno, &mp, parent_txnid);

  const page_type_t type = walk_page_type(mp);
  const size_t nentries = mp ? page_numkeys(mp) : 0;
  size_t header_size = (mp && !is_dupfix_leaf(mp)) ? PAGEHDRSZ + mp->lower : PAGEHDRSZ;
  size_t payload_size = 0;
  size_t unused_size = (mp ? page_room(mp) : ctx->txn->env->ps - header_size) - payload_size;
  size_t align_bytes = 0;

  for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) {
    if (type == page_dupfix_leaf) {
      /* DUPFIX pages have no entries[] or node headers */
      payload_size += mp->dupfix_ksize;
      continue;
    }

    const node_t *node = page_node(mp, i);
    header_size += NODESIZE;
    const size_t node_key_size = node_ks(node);
    payload_size += node_key_size;

    if (type == page_branch) {
      ASSERT(i > 0 || node_ks(node) == 0);
      align_bytes += node_key_size & 1;
      continue;
    }

    const size_t node_data_size = node_ds(node);
    ASSERT(type == page_leaf);
    switch (node_flags(node)) {
    case 0 /* usual node */:
      payload_size += node_data_size;
      align_bytes += (node_key_size + node_data_size) & 1;
      break;

    case N_BIG /* long data on the large/overflow page */:
      ASSERT(err == MDBX_SUCCESS);
      payload_size += sizeof(pgno_t);
      align_bytes += node_key_size & 1;
      break;

    case N_TREE /* sub-db */:
      if (unlikely(node_data_size != sizeof(tree_t))) {
        ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", (unsigned)node_data_size);
        ASSERT(err == MDBX_CORRUPTED);
        err = MDBX_CORRUPTED;
      }
      header_size += node_data_size;
      align_bytes += (node_key_size + node_data_size) & 1;
      break;

    case N_TREE | N_DUP /* dupsorted sub-tree */:
      if (unlikely(node_data_size != sizeof(tree_t))) {
        ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_data_size);
        ASSERT(err == MDBX_CORRUPTED);
        err = MDBX_CORRUPTED;
      }
      header_size += node_data_size;
      align_bytes += (node_key_size + node_data_size) & 1;
      break;

    case N_DUP /* short sub-page */: {
      if (unlikely(node_data_size <= PAGEHDRSZ || (node_data_size & 1))) {
        ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page node size", (unsigned)node_data_size);
        ASSERT(err == MDBX_CORRUPTED);
        err = MDBX_CORRUPTED;
        break;
      }

      const page_t *const sp = node_data(node);
      const page_type_t subtype = walk_subpage_type(sp);
      const size_t nsubkeys = page_numkeys(sp);
      if (unlikely(subtype == page_sub_broken)) {
        ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page flags", sp->flags);
        ASSERT(err == MDBX_CORRUPTED);
        err = MDBX_CORRUPTED;
      }

      size_t subheader_size = is_dupfix_leaf(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->lower;
      size_t subunused_size = page_room(sp);
      size_t subpayload_size = 0;
      size_t subalign_bytes = 0;

      if (err == MDBX_SUCCESS)
        for (size_t ii = 0; ii < nsubkeys; ++ii) {
          if (subtype == page_sub_dupfix_leaf) {
            /* DUPFIX pages have no entries[] or node headers */
            subpayload_size += sp->dupfix_ksize;
          } else {
            ASSERT(subtype == page_sub_leaf);
            const node_t *subnode = page_node(sp, ii);
            const size_t subnode_size = node_ks(subnode) + node_ds(subnode);
            subheader_size += NODESIZE;
            subpayload_size += subnode_size;
            subalign_bytes += subnode_size & 1;
            if (unlikely(node_flags(subnode) != 0)) {
              ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "unexpected sub-node flags",
                    node_flags(subnode));
              ASSERT(err == MDBX_CORRUPTED);
              err = MDBX_CORRUPTED;
            }
          }
        }

      header_size += subheader_size;
      unused_size += subunused_size;
      payload_size += subpayload_size;
      align_bytes += subalign_bytes + (node_key_size & 1);
    } break;

    default:
      ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node flags", node_flags(node));
      ASSERT(err == MDBX_CORRUPTED);
      err = MDBX_CORRUPTED;
    }
  }

  int rc = ctx->visitor(pgno, 1, ctx->userctx, ctx->deep, tbl, ctx->txn->env->ps, type, mp->txnid, err, nentries,
                        payload_size, header_size, unused_size + align_bytes, parent_pgno);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) {
    if (type == page_dupfix_leaf)
      continue;

    node_t *node = page_node(mp, i);
    if (type == page_branch) {
      ASSERT(err == MDBX_SUCCESS);
      ctx->deep += 1;
      rc = walk_pgno(ctx, tbl, node_pgno(node), mp->txnid, pgno);
      ctx->deep -= 1;
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
      continue;
    }

    const size_t node_data_size = node_ds(node);
    ASSERT(type == page_leaf);
    switch (node_flags(node)) {
    default:
      continue;

    case N_BIG /* long data on the large/overflow page */: {
      const pgno_t large_pgno = node_largedata_pgno(node);
      const size_t over_payload = node_data_size;
      const size_t over_header = PAGEHDRSZ;

      ASSERT(err == MDBX_SUCCESS);
      pgr_t lp = page_get_large(ctx->cursor, large_pgno, mp->txnid);
      const size_t npages = ((err = lp.err) == MDBX_SUCCESS) ? lp.page->pages : 1;
      const size_t pagesize = pgno2bytes(ctx->txn->env, npages);
      const size_t over_unused = pagesize - over_payload - over_header;
      rc = ctx->visitor(large_pgno, npages, ctx->userctx, ctx->deep + 1, tbl, pagesize, page_large, lp.page->txnid, err,
                        1, over_payload, over_header, over_unused, pgno);
      if (unlikely(rc != MDBX_SUCCESS))
        return rc;
    } break;

    case N_TREE /* sub-db */:
      if (unlikely(node_data_size != sizeof(tree_t))) {
        ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_data_size);
        ASSERT(err == MDBX_CORRUPTED);
        err = MDBX_CORRUPTED;
      } else {
        tree_t aligned_db;
        memcpy(&aligned_db, node_data(node), sizeof(aligned_db));
        walk_tbl_t table = {{node_key(node), node_ks(node)}, nullptr, nullptr};
        table.internal = &aligned_db;
        ASSERT(err == MDBX_SUCCESS);
        ctx->deep += 1;
        rc = walk_tbl(ctx, &table, pgno);
        ctx->deep -= 1;
        if (unlikely(rc != MDBX_SUCCESS))
          return rc;
      }
      break;

    case N_TREE | N_DUP /* dupsorted sub-tree */:
      if (unlikely(node_data_size != sizeof(tree_t))) {
        ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size",
              (unsigned)node_data_size);
        ASSERT(err == MDBX_CORRUPTED);
        err = MDBX_CORRUPTED;
      } else {
        tree_t aligned_db;
        memcpy(&aligned_db, node_data(node), sizeof(aligned_db));
        ASSERT(err == MDBX_SUCCESS);
        err = cursor_dupsort_setup(ctx->cursor, node, mp);
        if (likely(err == MDBX_SUCCESS)) {
          ASSERT(ctx->cursor->subcur == &container_of(ctx->cursor, cursor_couple_t, outer)->inner);
          ctx->cursor = &ctx->cursor->subcur->cursor;
          ctx->deep += 1;
          tbl->nested = &aligned_db;
          rc = walk_pgno(ctx, tbl, aligned_db.root, mp->txnid, pgno);
          tbl->nested = nullptr;
          ctx->deep -= 1;
          subcur_t *inner_xcursor = container_of(ctx->cursor, subcur_t, cursor);
          cursor_couple_t *couple = container_of(inner_xcursor, cursor_couple_t, inner);
          ctx->cursor = &couple->outer;
          if (unlikely(rc != MDBX_SUCCESS))
            return rc;
        }
      }
      break;

    case N_DUP /* short sub-page */:
      if (unlikely(node_data_size <= PAGEHDRSZ || (node_data_size & 1))) {
        ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page node size", (unsigned)node_data_size);
        ASSERT(err == MDBX_CORRUPTED);
        err = MDBX_CORRUPTED;
      } else {
        const page_t *const sp = node_data(node);
        const page_type_t subtype = walk_subpage_type(sp);
        const size_t nsubkeys = page_numkeys(sp);
        if (unlikely(subtype == page_sub_broken)) {
          ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page flags", sp->flags);
          ASSERT(err == MDBX_CORRUPTED);
          err = MDBX_CORRUPTED;
        }

        size_t subheader_size = is_dupfix_leaf(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->lower;
        size_t subunused_size = page_room(sp);
        size_t subpayload_size = 0;
        size_t subalign_bytes = 0;

        for (size_t ii = 0; err == MDBX_SUCCESS && ii < nsubkeys; ++ii) {
          if (subtype == page_sub_dupfix_leaf) {
            /* DUPFIX pages have no entries[] or node headers */
            subpayload_size += sp->dupfix_ksize;
          } else {
            ASSERT(subtype == page_sub_leaf);
            const node_t *subnode = page_node(sp, ii);
            const size_t subnode_size = node_ks(subnode) + node_ds(subnode);
            subheader_size += NODESIZE;
            subpayload_size += subnode_size;
            subalign_bytes += subnode_size & 1;
            if (unlikely(node_flags(subnode) != 0)) {
              ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "unexpected sub-node flags",
                    node_flags(subnode));
              ASSERT(err == MDBX_CORRUPTED);
              err = MDBX_CORRUPTED;
            }
          }
        }

        rc = ctx->visitor(pgno, 0, ctx->userctx, ctx->deep + 1, tbl, node_data_size, subtype, sp->txnid, err, nsubkeys,
                          subpayload_size, subheader_size, subunused_size + subalign_bytes, pgno);
        if (unlikely(rc != MDBX_SUCCESS))
          return rc;
      }
      break;
    }
  }

  return err;
}

__cold int walk_tbl(walk_ctx_t *ctx, walk_tbl_t *tbl, pgno_t parent_page) {
  tree_t *const db = tbl->internal;
  if (unlikely(db->root == P_INVALID))
    return MDBX_SUCCESS; /* empty db */

  kvx_t kvx = {.clc = {.k = {.lmin = INT_MAX}, .v = {.lmin = INT_MAX}}};
  cursor_couple_t couple;
  int rc = cursor_init4walk(&couple, ctx->txn, db, &kvx);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  const uint8_t cursor_checking = (ctx->options & dont_check_keys_ordering) ? z_pagecheck | z_ignord : z_pagecheck;
  couple.outer.checking |= cursor_checking;
  couple.inner.cursor.checking |= cursor_checking;
  couple.outer.next = ctx->cursor;
  couple.outer.top_and_flags = z_disable_tree_search_fastpath;
  ctx->cursor = &couple.outer;
  txnid_t mod_txnid = db->mod_txnid;
  if (!mod_txnid || (db == &ctx->txn->dbs[FREE_DBI] && (ctx->txn->dbi_state[FREE_DBI] & DBI_DIRTY)) ||
      (db == &ctx->txn->dbs[MAIN_DBI] && (ctx->txn->dbi_state[MAIN_DBI] & DBI_DIRTY)))
    mod_txnid = ctx->txn->front_txnid;
  rc = walk_pgno(ctx, tbl, db->root, mod_txnid, parent_page);
  ctx->cursor = couple.outer.next;
  return rc;
}

__cold int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options) {
  int rc = check_txn(txn, MDBX_TXN_BLOCKED);
  if (unlikely(rc != MDBX_SUCCESS))
    return rc;

  walk_ctx_t ctx = {.txn = txn, .userctx = user, .visitor = visitor, .options = options};
  walk_tbl_t tbl = {.name = {.iov_base = MDBX_CHK_GC}, .internal = &txn->dbs[FREE_DBI]};
  if ((options & dont_walk_GC) == 0)
    rc = walk_tbl(&ctx, &tbl, 0);
  if ((options & dont_walk_MAIN) == 0 && rc == MDBX_SUCCESS) {
    tbl.name.iov_base = MDBX_CHK_MAIN;
    tbl.internal = &txn->dbs[MAIN_DBI];
    rc = walk_tbl(&ctx, &tbl, 0);
  }
  return rc;
}

#if defined(_WIN32) || defined(_WIN64)

//------------------------------------------------------------------------------
// Stub for slim read-write lock
// Portion Copyright (C) 1995-2002 Brad Wilson

static void WINAPI stub_srwlock_Init(osal_srwlock_t *srwl) { srwl->readerCount = srwl->writerCount = 0; }

static void WINAPI stub_srwlock_AcquireShared(osal_srwlock_t *srwl) {
  while (true) {
    ASSERT(srwl->writerCount >= 0 && srwl->readerCount >= 0);

    //  If there's a writer already, spin without unnecessarily
    //  interlocking the CPUs
    if (srwl->writerCount != 0) {
      SwitchToThread();
      continue;
    }

    //  Add to the readers list
    _InterlockedIncrement(&srwl->readerCount);

    // Check for writers again (we may have been preempted). If
    // there are no writers writing or waiting, then we're done.
    if (srwl->writerCount == 0)
      break;

    // Remove from the readers list, spin, try again
    _InterlockedDecrement(&srwl->readerCount);
    SwitchToThread();
  }
}

static void WINAPI stub_srwlock_ReleaseShared(osal_srwlock_t *srwl) {
  ASSERT(srwl->readerCount > 0);
  _InterlockedDecrement(&srwl->readerCount);
}

static void WINAPI stub_srwlock_AcquireExclusive(osal_srwlock_t *srwl) {
  while (true) {
    ASSERT(srwl->writerCount >= 0 && srwl->readerCount >= 0);

    //  If there's a writer already, spin without unnecessarily
    //  interlocking the CPUs
    if (srwl->writerCount != 0) {
      SwitchToThread();
      continue;
    }

    // See if we can become the writer (expensive, because it inter-
    // locks the CPUs, so writing should be an infrequent process)
    if (_InterlockedExchange(&srwl->writerCount, 1) == 0)
      break;
  }

  // Now we're the writer, but there may be outstanding readers.
  // Spin until there aren't any more; new readers will wait now
  // that we're the writer.
  while (srwl->readerCount != 0) {
    ASSERT(srwl->writerCount >= 0 && srwl->readerCount >= 0);
    SwitchToThread();
  }
}

static void WINAPI stub_srwlock_ReleaseExclusive(osal_srwlock_t *srwl) {
  ASSERT(srwl->writerCount == 1 && srwl->readerCount >= 0);
  srwl->writerCount = 0;
}

static uint64_t WINAPI stub_GetTickCount64(void) {
  LARGE_INTEGER Counter, Frequency;
  return (QueryPerformanceFrequency(&Frequency) && QueryPerformanceCounter(&Counter))
             ? Counter.QuadPart * 1000ul / Frequency.QuadPart
             : 0;
}

//------------------------------------------------------------------------------

struct libmdbx_imports imports;

#if __GNUC_PREREQ(8, 0)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wcast-function-type"
#endif /* GCC/MINGW */

#define MDBX_IMPORT(HANDLE, ENTRY) imports.ENTRY = (MDBX_##ENTRY)GetProcAddress(HANDLE, #ENTRY)

void windows_import(void) {
  const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll");
  if (hNtdll) {
    globals.running_under_Wine = !!GetProcAddress(hNtdll, "wine_get_version");
    if (!globals.running_under_Wine) {
      MDBX_IMPORT(hNtdll, NtFsControlFile);
      MDBX_IMPORT(hNtdll, NtExtendSection);
      ENSURE(imports.NtExtendSection);
    }
  }

  const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll");
  if (hKernel32dll) {
    MDBX_IMPORT(hKernel32dll, GetFileInformationByHandleEx);
    MDBX_IMPORT(hKernel32dll, GetTickCount64);
    if (!imports.GetTickCount64)
      imports.GetTickCount64 = stub_GetTickCount64;
    if (!globals.running_under_Wine) {
      MDBX_IMPORT(hKernel32dll, SetFileInformationByHandle);
      MDBX_IMPORT(hKernel32dll, GetVolumeInformationByHandleW);
      MDBX_IMPORT(hKernel32dll, GetFinalPathNameByHandleW);
      MDBX_IMPORT(hKernel32dll, PrefetchVirtualMemory);
      MDBX_IMPORT(hKernel32dll, SetFileIoOverlappedRange);
    }
  }

  const osal_srwlock_t_function srwlock_init =
      (osal_srwlock_t_function)(hKernel32dll ? GetProcAddress(hKernel32dll, "InitializeSRWLock") : nullptr);
  if (srwlock_init) {
    imports.srwl_Init = srwlock_init;
    imports.srwl_AcquireShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockShared");
    imports.srwl_ReleaseShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockShared");
    imports.srwl_AcquireExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockExclusive");
    imports.srwl_ReleaseExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockExclusive");
  } else {
    imports.srwl_Init = stub_srwlock_Init;
    imports.srwl_AcquireShared = stub_srwlock_AcquireShared;
    imports.srwl_ReleaseShared = stub_srwlock_ReleaseShared;
    imports.srwl_AcquireExclusive = stub_srwlock_AcquireExclusive;
    imports.srwl_ReleaseExclusive = stub_srwlock_ReleaseExclusive;
  }

  const HINSTANCE hAdvapi32dll = GetModuleHandleA("advapi32.dll");
  if (hAdvapi32dll) {
    MDBX_IMPORT(hAdvapi32dll, RegGetValueA);
  }

  const HINSTANCE hOle32dll = GetModuleHandleA("ole32.dll");
  if (hOle32dll) {
    MDBX_IMPORT(hOle32dll, CoCreateGuid);
  }
}

#undef MDBX_IMPORT

#if __GNUC_PREREQ(8, 0)
#pragma GCC diagnostic pop
#endif /* GCC/MINGW */

#endif /* Windows */
/* This is CMake-template for libmdbx's version.c
 ******************************************************************************/

#if !defined(MDBX_VERSION_UNSTABLE) &&                                                                                 \
    (MDBX_VERSION_MAJOR != 0 || MDBX_VERSION_MINOR != 14)
#error "API version mismatch! Had `git fetch --tags` done?"
#endif

static const char sourcery[] =
#ifdef MDBX_VERSION_UNSTABLE
    "UNSTABLE@"
#endif
    MDBX_STRINGIFY(MDBX_BUILD_SOURCERY);

__dll_export
#ifdef __attribute_used__
    __attribute_used__
#elif defined(__GNUC__) || __has_attribute(__used__)
    __attribute__((__used__))
#endif
#ifdef __attribute_externally_visible__
        __attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__)
    __attribute__((__externally_visible__))
#endif
    const struct MDBX_version_info mdbx_version = {
        0,
        14,
        2,
        0,
        "", /* pre-release suffix of SemVer
                                        0.14.2 */
        {"2026-05-14T21:14:59+03:00", "171ce5282b63a650673f1077f40ed5ed80a2af11", "530d02658f6c692cad9542b295c4cd7afeac0d4f", "v0.14.2-0-g530d0265"},
        sourcery};

__dll_export
#ifdef __attribute_used__
    __attribute_used__
#elif defined(__GNUC__) || __has_attribute(__used__)
    __attribute__((__used__))
#endif
#ifdef __attribute_externally_visible__
        __attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__)
    __attribute__((__externally_visible__))
#endif
    const char *const mdbx_sourcery_anchor = sourcery;
