Dynamically loading xNVMe

In this seciton, you will find two examples of dynamically loading the xNVMe shared library and via it utilize the C along with notes on common pitfalls when doing so.

The example code demonstrates how to utilize xnvme_enumerate with callback function, callback arguments and associated data-structures and API functions. The example will first be presented in C and then in Python.

In C

Here dlopen() is used to load the library and pointers to the relevant functions are returned by dlsym().

#include <dlfcn.h>
#include <libxnvme.h>
#include <stdio.h>
#include <stdlib.h>

struct lib {
	void *handle;

	struct xnvme_opts (*opts_default)(void);
	struct xnvme_ident *(*get_ident)(struct xnvme_dev *);
	int (*ident_pr)(const struct xnvme_ident *, int);
	int (*enumerate)(char *, struct xnvme_opts *, xnvme_enumerate_cb, void *);
};

int
enum_cb(struct xnvme_dev *dev, void *cb_args)
{
	struct lib *lib = cb_args;

	lib->ident_pr(lib->get_ident(dev), XNVME_PR_DEF);

	return XNVME_ENUMERATE_DEV_CLOSE;
}

int
main(int argc, char *argv[])
{
	struct lib lib = {0};
	struct xnvme_opts opts;
	int err;

	if (argc < 2) {
		printf("usage: %s <library_path> [sys_uri]\n", argv[0]);
		return EXIT_FAILURE;
	}

	lib.handle = dlopen(argv[1], RTLD_LAZY);
	if (!lib.handle) {
		fprintf(stderr, "%s\n", dlerror());
		return EXIT_FAILURE;
	}

	lib.enumerate = dlsym(lib.handle, "xnvme_enumerate");
	if (!lib.enumerate) {
		fprintf(stderr, "%s\n", dlerror());
		err = EXIT_FAILURE;
		goto exit;
	}

	lib.opts_default = dlsym(lib.handle, "xnvme_opts_default");
	if (!lib.opts_default) {
		fprintf(stderr, "%s\n", dlerror());
		err = EXIT_FAILURE;
		goto exit;
	}

	lib.ident_pr = dlsym(lib.handle, "xnvme_ident_pr");
	if (!lib.ident_pr) {
		fprintf(stderr, "%s\n", dlerror());
		err = EXIT_FAILURE;
		goto exit;
	}

	lib.get_ident = dlsym(lib.handle, "xnvme_dev_get_ident");
	if (!lib.get_ident) {
		fprintf(stderr, "%s\n", dlerror());
		err = EXIT_FAILURE;
		goto exit;
	}

	opts = lib.opts_default();

	err = lib.enumerate(argc > 2 ? argv[2] : NULL, &opts, enum_cb, &lib);
	if (err) {
		printf("FAILED: enumerate(), err: %d", err);
		err = EXIT_FAILURE;
	}

exit:
	dlclose(lib.handle);

	return err;
}

Note

When dynamically loading a library then you can only load symbols, such as functions. Entities such as type-definitions, enums, functions declared static inline, MACROs etc. are not available as loadable symbols. However, these can be obtained by importing the respective headers.

Note

In the above example most of the code is “boiler-plate” doing error-handling when loading functions. This might seem a bit excessive, however, it is required for a language such as C, since there are no exceptions being raised upon error.

Try compiling the above example, e.g. on a Linux system with gcc do:

gcc ../tutorial/dynamic_loading/enumerate_example.c -ldl -o enumerate_example

This will produce an executable named enumerate_example. You can run it by executing:

./enumerate_example $(pkg-config xnvme --variable=libdir)/libxnvme.so

The should produce output similar to:

xnvme_ident:
  uri: '/dev/nvme0'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/nvme0n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/nvme0n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x2
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/nvme1'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:adcdbeef'
xnvme_ident:
  uri: '/dev/nvme1n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:adcdbeef'
xnvme_ident:
  uri: '/dev/nvme2'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:beefcace'
xnvme_ident:
  uri: '/dev/nvme2n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:beefcace'
xnvme_ident:
  uri: '/dev/nvme3'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:subsys0'
xnvme_ident:
  uri: '/dev/nvme3n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:subsys0'
xnvme_ident:
  uri: '/dev/nvme4'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n3'
  dtype: 0x2
  nsid: 0x3
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n4'
  dtype: 0x2
  nsid: 0x4
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n5'
  dtype: 0x2
  nsid: 0x5
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng0n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/ng0n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x2
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/ng0n3'
  dtype: 0x2
  nsid: 0x3
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/ng1n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:adcdbeef'
xnvme_ident:
  uri: '/dev/ng2n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:beefcace'
xnvme_ident:
  uri: '/dev/ng3n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:subsys0'
xnvme_ident:
  uri: '/dev/ng4n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n3'
  dtype: 0x2
  nsid: 0x3
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n4'
  dtype: 0x2
  nsid: 0x4
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n5'
  dtype: 0x2
  nsid: 0x5
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'

Note

pkg-config is used to locate where the shared-library. Make sure you have it and xNVMe installed on the system or just provide an absolute path to libxnvme-shared.so.

Note

If you see no output, then try running it as super-user or via sudo

In Python

Here ctypes is used to load the library. Unlike C, then each function does not need to be be explicitly loaded, additionally some assistance is given to locate the library.

import ctypes
import ctypes.util
import os
import subprocess


def load_capi():
    """Attempt to load the library via ctypes"""

    def search_paths():
        """Generate paths to try and load"""

        for search in ["xnvme"]:
            path = ctypes.util.find_library(search)
            if path:
                yield path

        try:
            proc = subprocess.run(
                ["pkg-config", "xnvme", "--variable=libdir"],
                check=True,
                capture_output=True,
            )
            if not proc.returncode:
                yield os.path.join(proc.stdout.decode("utf-8").strip(), "libxnvme.so")
        except subprocess.CalledProcessError:
            pass

    for spath in search_paths():
        try:
            lib = ctypes.CDLL(spath)
            if lib:
                return lib
        except OSError:
            continue

    return None


capi = load_capi()


class Ident(ctypes.Structure):
    """Struct containing device identifiers"""

    _fields_ = [
        ("uri", ctypes.c_char * 384),
        ("dtype", ctypes.c_uint32),
        ("nsid", ctypes.c_uint32),
        ("csi", ctypes.c_uint8),
        ("rsvd", ctypes.c_uint8 * 3),
    ]


class Dev(ctypes.Structure):
    """Struct representing the device"""

    _fields_: list[tuple] = []


def enum_cb(dev):
    """Callback function invoked for each device"""

    get_ident = capi.xnvme_dev_get_ident
    get_ident.restype = ctypes.POINTER(Ident)
    capi.xnvme_ident_pr(get_ident(dev), 0x0)

    return 1


def main():
    """Enumerate devices on the system"""

    enum_cb_type = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(Dev))
    capi.xnvme_enumerate(None, None, enum_cb_type(enum_cb), None)


if __name__ == "__main__":
    main()

Note

In case ctypes.util.find_library()/ctypes.CDLL() cannot find/load the library then try providing an absolute path to the library instead. In case you have pkg-config installed then it can help you locate the library.

Note

With ctypes C structs are mapped to Python via ctypes.Structure classes. The order in which the fields are declared determines which member they are mapped to, not their name.

The example can be run like so:

python3 ../tutorial/dynamic_loading/enumerate_example.py

The command should produce output similar to:

xnvme_ident:
  uri: '/dev/nvme0'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/nvme0n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/nvme0n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x2
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/nvme1'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:adcdbeef'
xnvme_ident:
  uri: '/dev/nvme1n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:adcdbeef'
xnvme_ident:
  uri: '/dev/nvme2'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:beefcace'
xnvme_ident:
  uri: '/dev/nvme2n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:beefcace'
xnvme_ident:
  uri: '/dev/nvme3'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:subsys0'
xnvme_ident:
  uri: '/dev/nvme3n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:subsys0'
xnvme_ident:
  uri: '/dev/nvme4'
  dtype: 0x1
  nsid: 0xffffffff
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n3'
  dtype: 0x2
  nsid: 0x3
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n4'
  dtype: 0x2
  nsid: 0x4
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/nvme4n5'
  dtype: 0x2
  nsid: 0x5
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng0n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/ng0n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x2
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/ng0n3'
  dtype: 0x2
  nsid: 0x3
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:deadbeef'
xnvme_ident:
  uri: '/dev/ng1n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:adcdbeef'
xnvme_ident:
  uri: '/dev/ng2n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:beefcace'
xnvme_ident:
  uri: '/dev/ng3n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:subsys0'
xnvme_ident:
  uri: '/dev/ng4n1'
  dtype: 0x2
  nsid: 0x1
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n2'
  dtype: 0x2
  nsid: 0x2
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n3'
  dtype: 0x2
  nsid: 0x3
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n4'
  dtype: 0x2
  nsid: 0x4
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'
xnvme_ident:
  uri: '/dev/ng4n5'
  dtype: 0x2
  nsid: 0x5
  csi: 0x0
  subnqn: 'nqn.2019-08.org.qemu:feebdaed'

Note

If you see no output, then try running it as super-user or via sudo