Agreed. I lazy-load NVM to get around that:

  lazy_nvm() {
    unset -f nvm node npm npx
    [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
  }
  nvm()  { lazy_nvm; nvm "$@"; }
  node() { lazy_nvm; node "$@"; }
  npm()  { lazy_nvm; npm "$@"; }
  npx()  { lazy_nvm; npx "$@"; }