feat: adds migration api and queries (#503)

* feat: adds migration api and queries

* fixes

* fixes

* removes unnecessary query

* refactor, starts adding tests

* fixes

* fixes

* adds remaining tests for passwordhashes

* fixes

* fixes comment

* udpates CHANGELOG and dependencies

* minor fixes

* implements some feedback

* comment fixes

* fixes

* fix

* implements feedback

* updates CHANGELOG.md and versio

* fixes

* adds hashingAlgorithm field param

* fixes

* implements feedback

* fixes

* test fixes

* marks function as testonly

* adds test for checking hahsing algorithm with lowercase and upper case names

* remvoes unnecessary enum

* feat: Adds support to migrate users from Firebase (#507)

* progress on adding firebase scrypt

* implements feedback

* removes unnecessary spacing

* adds poolsize for firebase scrypt hashing

* removes unnecessary file

* removes unnecessary statements

* adds some feedback

* fixes configs

* adds comment

* implements feedback

* fixs comment

* adds more tests; test fixs

* adds test to check that not setting signer keys throws error

* adds more tests and fixes

* adds test for signing in when wrong signer key is set

* adds test for firebase scrypt validation with different config values

* adds tests for firebase scrypt pool checking

* fixs for test

* feedback changes

* test fixs

* adds additional test check

* updates CHANGELOG.md and fixs typo

* updates CHANGELOG.md

* updates CHANGELOG.md

* test fixs

* fixs

* updates query

* updates CHANGELOG.md

Author: jscyo

CHANGELOG.md

@@ -7,10 +7,26 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [unreleased]
 
+## [4.0.0] - 2022-09-19
+
 ### Added
 
+- EmailPassword User migration API which allows you to import users with their email and password hashes.
+- Support to import users with password hashes from Firebase
+- Support with CDI version `2.16`
 - Hello API on `/` route.
 
+### Database Changes
+- Updates the `password_hash` column in the `emailpassword_users` table from `VARCHAR(128)` to `VARCHAR(256)` to support more password  hash lengths.
+- Updates the `third_party_user_id` column in the `thirdparty_users` table from `VARCHAR(128)` to `VARCHAR(256)` to resolve https://github.com/supertokens/supertokens-core/issues/306
+
+- For legacy users who are self hosting the SuperTokens core run the following command to update your database with the changes:
+  - With MySql:
+    `ALTER TABLE thirdparty_users MODIFY third_party_user_id VARCHAR(256); ALTER TABLE emailpassword_users MODIFY password_hash VARCHAR(256);` 
+  - With PostgreSQL:
+    `ALTER TABLE thirdparty_users MODIFY third_party_user_id VARCHAR(256); ALTER TABLE emailpassword_users MODIFY password_hash VARCHAR(256);`   
+    
+
 ## [3.16.2] - 2022-09-02
 
 ### Bug fixes

build.gradle

@@ -19,7 +19,7 @@ compileTestJava { options.encoding = "UTF-8" }
 //    }
 //}
 
-version = "3.16.2"
+version = "4.0.0"
 
 
 repositories {
@@ -61,6 +61,9 @@ dependencies {
     // https://mvnrepository.com/artifact/com.auth0/java-jwt
     implementation 'com.auth0:java-jwt:4.0.0'
 
+    // https://mvnrepository.com/artifact/com.lambdaworks/scrypt
+    implementation group: 'com.lambdaworks', name: 'scrypt', version: '1.4.0'
+
     compileOnly project(":supertokens-plugin-interface")
     testImplementation project(":supertokens-plugin-interface")
 

config.yaml

@@ -105,4 +105,10 @@ core_config_version: 0
 # argon2_hashing_pool_size:
 
 # (OPTIONAL | Default: "INFO"). Logging level for the core. Values are "DEBUG" | "INFO" | "WARN" | "ERROR" | "NONE"
-# log_level:
+# log_level:
+
+# (OPTIONAL | Default: null). The signer key used for firebase scrypt password hashing
+# firebase_password_hashing_signer_key:
+
+# (OPTIONAL | Default: 1). Number of concurrent firebase scrypt hashes that can happen at the same time for sign in requests.
+# firebase_password_hashing_pool_size:

coreDriverInterfaceSupported.json

@@ -9,6 +9,7 @@
     "2.12",
     "2.13",
     "2.14",
-    "2.15"
+    "2.15",
+    "2.16"
   ]
 }

devConfig.yaml

@@ -105,4 +105,10 @@ disable_telemetry: true
 # argon2_hashing_pool_size:
 
 # (OPTIONAL | Default: "INFO"). Logging level for the core. Values are "DEBUG" | "INFO" | "WARN" | "ERROR" | "NONE"
-# log_level:
+# log_level:
+
+# (OPTIONAL | Default: null). The signer key used for firebase scrypt password hashing
+# firebase_password_hashing_signer_key:
+
+# (OPTIONAL | Default: 1). Number of concurrent firebase scrypt hashes that can happen at the same time for sign in requests.
+# firebase_password_hashing_pool_size:

implementationDependencies.json

@@ -95,6 +95,11 @@
       "jar": "https://repo1.maven.org/maven2/net/java/dev/jna/jna/5.8.0/jna-5.8.0.jar",
       "name": "JNA 5.8.0",
       "src": "https://repo1.maven.org/maven2/net/java/dev/jna/jna/5.8.0/jna-5.8.0-sources.jar"
+    },
+    {
+      "jar": "https://repo1.maven.org/maven2/com/lambdaworks/scrypt/1.4.0/scrypt-1.4.0-javadoc.jar",
+      "name": "Scrypt 1.4.0",
+      "src": "https://repo1.maven.org/maven2/com/lambdaworks/scrypt/1.4.0/scrypt-1.4.0-sources.jar"
     }
   ]
 }

src/main/java/io/supertokens/ProcessState.java

@@ -77,7 +77,8 @@ public enum PROCESS_STATE {
         INIT, INIT_FAILURE, STARTED, SHUTTING_DOWN, STOPPED, RETRYING_ACCESS_TOKEN_JWT_VERIFICATION,
         CRON_TASK_ERROR_LOGGING, WAITING_TO_INIT_STORAGE_MODULE, GET_SESSION_NEW_TOKENS, DEADLOCK_FOUND,
         CREATING_NEW_TABLE, SENDING_TELEMETRY, SENT_TELEMETRY, SETTING_ACCESS_TOKEN_SIGNING_KEY_TO_NULL,
-        PASSWORD_HASH_BCRYPT, PASSWORD_HASH_ARGON, PASSWORD_VERIFY_BCRYPT, PASSWORD_VERIFY_ARGON
+        PASSWORD_HASH_BCRYPT, PASSWORD_HASH_ARGON, PASSWORD_VERIFY_BCRYPT, PASSWORD_VERIFY_ARGON,
+        PASSWORD_VERIFY_FIREBASE_SCRYPT
     }
 
     public static class EventAndException {

src/main/java/io/supertokens/config/CoreConfig.java

@@ -99,6 +99,9 @@ public class CoreConfig {
     @JsonProperty
     private int argon2_hashing_pool_size = 1;
 
+    @JsonProperty
+    private int firebase_password_hashing_pool_size = 1;
+
     @JsonProperty
     private int bcrypt_log_rounds = 11;
 
@@ -116,6 +119,9 @@ public class CoreConfig {
     @JsonProperty
     private String log_level = "INFO";
 
+    @JsonProperty
+    private String firebase_password_hashing_signer_key = null;
+
     private Set<LOG_LEVEL> allowedLogLevels = null;
 
     public Set<LOG_LEVEL> getLogLevels(Main main) {
@@ -161,7 +167,7 @@ public String getBasePath() {
     }
 
     public enum PASSWORD_HASHING_ALG {
-        ARGON2, BCRYPT
+        ARGON2, BCRYPT, FIREBASE_SCRYPT
     }
 
     public int getArgon2HashingPoolSize() {
@@ -172,6 +178,10 @@ public int getArgon2HashingPoolSize() {
         return Math.max(1, argon2_hashing_pool_size);
     }
 
+    public int getFirebaseSCryptPasswordHashingPoolSize() {
+        return Math.max(1, firebase_password_hashing_pool_size);
+    }
+
     public int getArgon2Iterations() {
         return argon2_iterations;
     }
@@ -188,6 +198,13 @@ public int getArgon2Parallelism() {
         return argon2_parallelism;
     }
 
+    public String getFirebase_password_hashing_signer_key() {
+        if (firebase_password_hashing_signer_key == null) {
+            throw new IllegalStateException("'firebase_password_hashing_signer_key' cannot be null");
+        }
+        return firebase_password_hashing_signer_key;
+    }
+
     public PASSWORD_HASHING_ALG getPasswordHashingAlg() {
         return PASSWORD_HASHING_ALG.valueOf(password_hashing_alg.toUpperCase());
     }

src/main/java/io/supertokens/emailpassword/EmailPassword.java

@@ -19,7 +19,9 @@
 import io.supertokens.Main;
 import io.supertokens.authRecipe.UserPaginationToken;
 import io.supertokens.config.Config;
+import io.supertokens.config.CoreConfig;
 import io.supertokens.emailpassword.exceptions.ResetPasswordInvalidTokenException;
+import io.supertokens.emailpassword.exceptions.UnsupportedPasswordHashingFormatException;
 import io.supertokens.emailpassword.exceptions.WrongCredentialsException;
 import io.supertokens.pluginInterface.emailpassword.PasswordResetTokenInfo;
 import io.supertokens.pluginInterface.emailpassword.UserInfo;
@@ -36,12 +38,23 @@
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import javax.servlet.ServletException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.spec.InvalidKeySpecException;
 
 public class EmailPassword {
 
+    public static class ImportUserResponse {
+        public boolean didUserAlreadyExist;
+        public UserInfo user;
+
+        public ImportUserResponse(boolean didUserAlreadyExist, UserInfo user) {
+            this.didUserAlreadyExist = didUserAlreadyExist;
+            this.user = user;
+        }
+    }
+
     @TestOnly
     public static long getPasswordResetTokenLifetimeForTests(Main main) {
         return getPasswordResetTokenLifetime(main);
@@ -76,6 +89,46 @@ public static UserInfo signUp(Main main, @Nonnull String email, @Nonnull String
         }
     }
 
+    public static ImportUserResponse importUserWithPasswordHash(Main main, @Nonnull String email,
+            @Nonnull String passwordHash, @Nullable CoreConfig.PASSWORD_HASHING_ALG hashingAlgorithm)
+            throws StorageQueryException, StorageTransactionLogicException, UnsupportedPasswordHashingFormatException {
+
+        PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(main, passwordHash, hashingAlgorithm);
+
+        while (true) {
+            String userId = Utils.getUUID();
+            long timeJoined = System.currentTimeMillis();
+
+            UserInfo userInfo = new UserInfo(userId, email, passwordHash, timeJoined);
+            EmailPasswordSQLStorage storage = StorageLayer.getEmailPasswordStorage(main);
+
+            try {
+                StorageLayer.getEmailPasswordStorage(main).signUp(userInfo);
+                return new ImportUserResponse(false, userInfo);
+            } catch (DuplicateUserIdException e) {
+                // we retry with a new userId
+            } catch (DuplicateEmailException e) {
+                UserInfo userInfoToBeUpdated = StorageLayer.getEmailPasswordStorage(main).getUserInfoUsingEmail(email);
+                // if user does not exist we retry signup
+                if (userInfoToBeUpdated != null) {
+                    String finalPasswordHash = passwordHash;
+                    storage.startTransaction(con -> {
+                        storage.updateUsersPassword_Transaction(con, userInfoToBeUpdated.id, finalPasswordHash);
+                        return null;
+                    });
+                    return new ImportUserResponse(true, userInfoToBeUpdated);
+                }
+            }
+        }
+    }
+
+    @TestOnly
+    public static ImportUserResponse importUserWithPasswordHash(Main main, @Nonnull String email,
+            @Nonnull String passwordHash)
+            throws StorageQueryException, StorageTransactionLogicException, UnsupportedPasswordHashingFormatException {
+        return importUserWithPasswordHash(main, email, passwordHash, null);
+    }
+
     public static UserInfo signIn(Main main, @Nonnull String email, @Nonnull String password)
             throws StorageQueryException, WrongCredentialsException {
 
@@ -91,6 +144,12 @@ public static UserInfo signIn(Main main, @Nonnull String email, @Nonnull String
             }
         } catch (WrongCredentialsException e) {
             throw e;
+        } catch (IllegalStateException e) {
+            if (e.getMessage().equals("'firebase_password_hashing_signer_key' cannot be null")) {
+                throw e;
+            }
+            throw new WrongCredentialsException();
+
         } catch (Exception ignored) {
             throw new WrongCredentialsException();
         }

src/main/java/io/supertokens/emailpassword/ParsedFirebaseSCryptResponse.java

@@ -0,0 +1,80 @@
+/*
+ *    Copyright (c) 2022, VRAI Labs and/or its affiliates. All rights reserved.
+ *
+ *    This software is licensed under the Apache License, Version 2.0 (the
+ *    "License") as published by the Apache Software Foundation.
+ *
+ *    You may not use this file except in compliance with the License. You may
+ *    obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ */
+
+package io.supertokens.emailpassword;
+
+public class ParsedFirebaseSCryptResponse {
+    public String passwordHash;
+    public String salt;
+    public String saltSeparator;
+    public int rounds;
+    public int memCost;
+
+    public static final String FIREBASE_SCRYPT_PREFIX = "f_scrypt";
+    private static final String FIREBASE_SCRYPT_SEPARATOR = "\\$";
+    private static final String FIREBASE_SCRYPT_MEM_COST_SEPARATOR = "m=";
+    private static final String FIREBASE_SCRYPT_ROUNDS_SEPARATOR = "r=";
+    private static final String FIREBASE_SCRYPT_SALT_SEPARATOR = "s=";
+
+    public ParsedFirebaseSCryptResponse(String passwordHash, String salt, String saltSeparator, int rounds,
+            int memCost) {
+        this.passwordHash = passwordHash;
+        this.salt = salt;
+        this.saltSeparator = saltSeparator;
+        this.rounds = rounds;
+        this.memCost = memCost;
+    }
+
+    public static ParsedFirebaseSCryptResponse fromHashString(String hash) {
+        try {
+            String[] separatedPasswordHash = hash.split(FIREBASE_SCRYPT_SEPARATOR);
+
+            // check that stored password hash contains 7 fields and after splitting, the first field is empty and the
+            // second field has the firebase scrypt prefix
+            if (!(separatedPasswordHash.length == 7 && separatedPasswordHash[0].equals("")
+                    && separatedPasswordHash[1].equals(FIREBASE_SCRYPT_PREFIX))) {
+                return null;
+            }
+
+            String passwordHash = separatedPasswordHash[2];
+            String salt = separatedPasswordHash[3];
+            String saltSeparator = null;
+            Integer memCost = null;
+            Integer rounds = null;
+
+            for (int i = 4; i < separatedPasswordHash.length; i++) {
+                if (separatedPasswordHash[i].startsWith(FIREBASE_SCRYPT_MEM_COST_SEPARATOR)) {
+                    memCost = Integer.parseInt(separatedPasswordHash[i].split(FIREBASE_SCRYPT_MEM_COST_SEPARATOR)[1]);
+                    continue;
+                }
+                if (separatedPasswordHash[i].startsWith(FIREBASE_SCRYPT_ROUNDS_SEPARATOR)) {
+                    rounds = Integer.parseInt(separatedPasswordHash[i].split(FIREBASE_SCRYPT_ROUNDS_SEPARATOR)[1]);
+                    continue;
+                }
+                if (separatedPasswordHash[i].startsWith(FIREBASE_SCRYPT_SALT_SEPARATOR)) {
+                    saltSeparator = separatedPasswordHash[i].split(FIREBASE_SCRYPT_SALT_SEPARATOR)[1];
+                }
+            }
+
+            if (passwordHash == null || salt == null || saltSeparator == null || memCost == null || rounds == null) {
+                return null;
+            }
+            return new ParsedFirebaseSCryptResponse(passwordHash, salt, saltSeparator, rounds, memCost);
+        } catch (Throwable e) {
+            return null;
+        }
+    }
+}