Logging error reason of failures in qrmi_resource_new(), qrmi_resource_is_accessible() etc. (#149)

* Logging resource_new() error reason

* add QRMI_PLUGIN_ERROR envvar

* keep_if_exists

* remove func name from log message

* call qrmi_get_last_error()

* use actual qrmi_config.json path

Author: ohtanim

plugins/spank_qrmi/spank_qrmi.c

@@ -51,7 +51,7 @@ static List g_acquired_resources = NULL;
 static qpu_resource_t *_acquired_resource_create(char *name, QrmiResourceType type,
                                                  const char *token);
 static void acquired_resource_destroy(void *object);
-static qpu_resource_t *_acquire_qpu(char *name, QrmiResourceType type);
+static qpu_resource_t *_acquire_qpu(spank_t spank_ctxt, char *name, QrmiResourceType type);
 static void _release_qpu(qpu_resource_t *res);
 
 /*
@@ -207,8 +207,8 @@ int slurm_spank_init_post_opt(spank_t spank_ctxt, int argc, char **argv) {
     while ((token = strtok_r(rest, ",", &rest))) {
         QrmiResourceDef *res = qrmi_config_resource_def_get(cnf, token);
         if (res != NULL) {
-            slurm_debug("%s: name(%s), type(%d) found in qrmi_config",
-                        plugin_name, res->name, res->type);
+            slurm_debug("%s: name(%s), type(%d) found in %s",
+                        plugin_name, res->name, res->type, argv[0]);
             /*
              * If user specifies access details in environment variables,
              * these are available as job environment variables. Reads through
@@ -261,7 +261,7 @@ int slurm_spank_init_post_opt(spank_t spank_ctxt, int argc, char **argv) {
             /*
              * Acquire QPU resource.
              */
-            qpu_resource_t *acquired = _acquire_qpu(res->name, res->type);
+            qpu_resource_t *acquired = _acquire_qpu(spank_ctxt, res->name, res->type);
             if (acquired != NULL) {
                 slurm_list_append(g_acquired_resources, acquired);
                 qrmi_buf_envvarname_for_res_create(&keybuf, res->name,
@@ -278,7 +278,7 @@ int slurm_spank_init_post_opt(spank_t spank_ctxt, int argc, char **argv) {
             }
             qrmi_config_resource_def_free(res);
         } else {
-            slurm_debug("resource %s not found.", token);
+            slurm_error("resource %s not found in %s", token, argv[0]);
         }
     }
     free(bufp);
@@ -544,35 +544,42 @@ static void acquired_resource_destroy(void *object) {
  * Acquire QPU resource specified by `name` and `type`. Returns
  * qpu_resource_t object if succeeded.
  */
-static qpu_resource_t *_acquire_qpu(char *name, QrmiResourceType type) {
-    qpu_resource_t *record = NULL;
+static qpu_resource_t *_acquire_qpu(spank_t spank_ctxt, char *name, QrmiResourceType type) {
     char *acquisition_token = NULL;
     bool is_accessible = false;
     QrmiReturnCode rc;
+    const char* last_error = NULL;
 
     void *qrmi = qrmi_resource_new(name, type);
-    if (qrmi != NULL) {
-        slurm_debug("%s, qrmi: %p", plugin_name, qrmi);
-        rc = qrmi_resource_is_accessible(qrmi, &is_accessible);
-        if ((rc != QRMI_RETURN_CODE_SUCCESS) || (is_accessible == false)) {
-            slurm_error("%s, %s is not accessible", plugin_name, name);
-            //functbl->free(qrmi);
-            qrmi_resource_free(qrmi);
-            return NULL;
-        }
-        rc = qrmi_resource_acquire(qrmi, &acquisition_token);
-        if ((rc == QRMI_RETURN_CODE_SUCCESS) && (acquisition_token != NULL)) {
-            slurm_debug("%s, acquisition_token: %s", plugin_name,
-                        acquisition_token);
-            record = _acquired_resource_create(name, type, acquisition_token);
-        }
+    if (qrmi == NULL) {
+        last_error = qrmi_get_last_error();
+        slurm_error("%s, %s", plugin_name, last_error);
+        spank_setenv(spank_ctxt, "QRMI_PLUGIN_ERROR", last_error, KEEP_IF_EXISTS);
+        qrmi_string_free((char*)last_error);
+        return NULL;
+    }
+
+    slurm_debug("%s, qrmi: %p", plugin_name, qrmi);
+    rc = qrmi_resource_is_accessible(qrmi, &is_accessible);
+    if ((rc != QRMI_RETURN_CODE_SUCCESS) || (is_accessible == false)) {
+        last_error = qrmi_get_last_error();
+        slurm_error("%s, %s is not accessible. %s", plugin_name, name, last_error);
+        qrmi_string_free((char*)last_error);
         qrmi_resource_free(qrmi);
-    } else {
-        slurm_error("%s/%s: Unsupported resource type: %d", plugin_name,
-                    __func__, type);
+        return NULL;
+    }
+    rc = qrmi_resource_acquire(qrmi, &acquisition_token);
+    qrmi_resource_free(qrmi);
+    if ((rc != QRMI_RETURN_CODE_SUCCESS) || (acquisition_token == NULL)) {
+        last_error = qrmi_get_last_error();
+        slurm_error("%s, resource acquisition failed: %s. %s", plugin_name, name, last_error);
+        spank_setenv(spank_ctxt, "QRMI_PLUGIN_ERROR", last_error, KEEP_IF_EXISTS);
+        qrmi_string_free((char*)last_error);
     }
 
-    return record;
+    slurm_debug("%s, acquisition_token: %s", plugin_name,
+                acquisition_token);
+    return _acquired_resource_create(name, type, acquisition_token);
 }
 
 /*
@@ -589,21 +596,29 @@ static void _release_qpu(qpu_resource_t *res) {
     slurm_debug("%s: releasing name(%s), type(%d), token(%s)", plugin_name,
                 res->name, res->type, res->acquisition_token);
     void *qrmi = qrmi_resource_new(res->name, res->type);
+    if (qrmi == NULL) {
+        const char* last_error = qrmi_get_last_error();
+        slurm_error("%s, %s", plugin_name, last_error);
+        qrmi_string_free((char*)last_error);
+        return;
+    }
     rc = qrmi_resource_release(qrmi, res->acquisition_token);
     if (rc != QRMI_RETURN_CODE_SUCCESS) {
-        slurm_error("%s: Failed to release acquired resource: name(%s), type(%d), token(%s)",
+        const char* last_error = qrmi_get_last_error();
+        slurm_error("%s, Failed to release acquired resource: name(%s), type(%d), token(%s), %s",
                     plugin_name,
-                    res->name, res->type, res->acquisition_token);
+                    res->name, res->type, res->acquisition_token, last_error);
+        qrmi_string_free((char*)last_error);
     }
     rc = qrmi_string_free(res->acquisition_token);
     if (rc != QRMI_RETURN_CODE_SUCCESS) {
-        slurm_error("%s: Failed to free acquisition token string: (%s)",
+        slurm_error("%s, Failed to free acquisition token string: (%s)",
                     plugin_name,
                     res->acquisition_token);
     }
     rc = qrmi_resource_free(qrmi);
     if (rc != QRMI_RETURN_CODE_SUCCESS) {
-        slurm_error("%s: Failed to free QrmiQuantumResource handle: (%p)",
+        slurm_error("%s, Failed to free QrmiQuantumResource handle: (%p)",
                     plugin_name,
                     qrmi);
     }