Do not always remove password field, but give other plugins the chance to keep password field by setting password enabled to active.

EventListener/RegistrationHandlerListener.php

@@ -46,14 +46,16 @@ class RegistrationHandlerListener implements EventSubscriberInterface
     {
         return [
             RegistrationEvents::BUILD_FORM => [
-                'onBuildForm',
+                [
+                    'onBuildFormSetPasswordEnabled',
                     10
                 ],
-            RegistrationEvents::HANDLE_FORM => [
                 [
-                    'onHandleFormSetPasswordEnabled',
-                    10
+                    'onBuildFormRemovePasswordField',
+                    -20
+                ]
             ],
+            RegistrationEvents::HANDLE_FORM => [
                 [
                     'onHandleFormConnectExternal',
                     0
@@ -83,10 +85,9 @@ class RegistrationHandlerListener implements EventSubscriberInterface
         return null;
     }
 
-    public function onBuildForm(RegistrationFormEvent $event)
+    public function onBuildFormSetPasswordEnabled(RegistrationFormEvent $event)
     {
         if ($user = $this->getTemporaryUser()) {
-            $event->getFormBuilder()->remove('password');
             $event->getFormBuilder()
                 ->getData()
                 ->setDisplayName($user->getDisplayName())
@@ -94,15 +95,20 @@ class RegistrationHandlerListener implements EventSubscriberInterface
         }
     }
 
-    public function onHandleFormSetPasswordEnabled(RegistrationHandleEvent $event)
+    public function onBuildFormRemovePasswordField(RegistrationFormEvent $event)
     {
-        if ($event->isFailed())
-            return;
-        $user = $event->getForm()->getData();
-        if (!$user)
-            return;
-        /* @var $user User */
-        $user->setPasswordEnabled(2);
+        if ($this->getTemporaryUser()) {
+            switch ($event->getFormBuilder()
+                ->getData()
+                ->getPasswordEnabled()) {
+                case 0:
+                case 2:
+                    $event->getFormBuilder()->remove('password');
+                    break;
+                case 1:
+                    break;
+            }
+        }
     }
 
     public function onHandleFormConnectExternal(RegistrationHandleEvent $event)