virsh-tool dump虚拟机内存源码分析

vlambda
2022-04-19

virsh-tool dump虚拟机内存源码分析

在之前的文章中，我们在分析时，尝试使用virsh dump虚拟机内存，以此来通过crash进行分析，因为virsh dump 虚拟机内存实际就是内存快照，和机器在crash时生成的vmcore文件是类似的。

virsh tools源码在libvirt项目的tools目录下，我们找到dump源码所在文件

tools/virsh-domain.c

 const vshCmdDef domManagementCmds[]{ {.name = "dump", .handler = cmdDump, .opts = opts_dump, .info = info_dump, .flags = 0 },   }

其中，domManagementCmds是个数组，里面包含了所有virsh命令参数信息。

处理dump的核心函数是cmdDump

static boolcmdDump(vshControl *ctl, const vshCmd *cmd) { virDomainPtr dom; bool verbose = false; const char *name = NULL;  const char *to = NULL;  virThread workerThread; g_autoptr(GMainContext) eventCtxt = g_main_context_new(); g_autoptr(GMainLoop) eventLoop = g_main_loop_new(eventCtxt, FALSE); virshCtrlData data = { .ctl = ctl, .cmd = cmd, .eventLoop = eventLoop, .ret = -1,  }; 
 if (!(dom = virshCommandOptDomain(ctl, cmd, &name))) return false;
 if (vshCommandOptStringReq(ctl, cmd, "file", &to) < 0) goto cleanup;
 if (vshCommandOptBool(cmd, "verbose")) verbose = true; 
 if (virThreadCreate(&workerThread, true, doDump, &data) < 0) goto cleanup;
 virshWatchJob(ctl, dom, verbose, eventLoop, &data.ret, 0, NULL, NULL, _("Dump"));
 virThreadJoin(&workerThread);
 if (!data.ret) vshPrintExtra(ctl, _("\nDomain '%s' dumped to %s\n"), name, to);
 cleanup: virshDomainFree(dom); return !data.ret;}

其中virThreadCreate会创建一个线程，线程函数是doDump，用来执行具体的dump操作，接下来看下doDump函数

static voiddoDump(void *opaque){ virshCtrlData *data = opaque; vshControl *ctl = data->ctl; const vshCmd *cmd = data->cmd; virDomainPtr dom = NULL; const char *name = NULL; const char *to = NULL;    ... if (vshCommandOptBool(cmd, "live")) flags |= VIR_DUMP_LIVE; if (vshCommandOptBool(cmd, "crash")) flags |= VIR_DUMP_CRASH; if (vshCommandOptBool(cmd, "bypass-cache")) flags |= VIR_DUMP_BYPASS_CACHE; if (vshCommandOptBool(cmd, "reset")) flags |= VIR_DUMP_RESET; if (vshCommandOptBool(cmd, "memory-only")) flags |= VIR_DUMP_MEMORY_ONLY; if (vshCommandOptStringQuiet(ctl, cmd, "format", &format) > 0) { if (STREQ(format, "kdump-zlib")) { dumpformat = VIR_DOMAIN_CORE_DUMP_FORMAT_KDUMP_ZLIB; } else if (STREQ(format, "kdump-lzo")) { dumpformat = VIR_DOMAIN_CORE_DUMP_FORMAT_KDUMP_LZO; } else if (STREQ(format, "kdump-snappy")) { dumpformat = VIR_DOMAIN_CORE_DUMP_FORMAT_KDUMP_SNAPPY;            } else if (STREQ(format, "elf")) {//dump出的文件格式是ELF dumpformat = VIR_DOMAIN_CORE_DUMP_FORMAT_RAW; } else { vshError(ctl, _("format '%s' is not supported, expecting " "'kdump-zlib', 'kdump-lzo', 'kdump-snappy' " "or 'elf'"), format); goto out; } }    }    ....     if (dumpformat != VIR_DOMAIN_CORE_DUMP_FORMAT_RAW) { if (virDomainCoreDumpWithFormat(dom, to, dumpformat, flags) < 0) { vshError(ctl, _("Failed to core dump domain '%s' to %s"), name, to); goto out; } } else { if (virDomainCoreDump(dom, to, flags) < 0) { vshError(ctl, _("Failed to core dump domain '%s' to %s"), name, to); goto out; } }....}

接下来看下virDomainCoreDump，其中参数dom是指向被dump内存的domain指针，to是dump到具体的位置,一般是文件名，flags是标识。

virDomainCoreDump源码位置在src/libvirt-domain.c文件中

intvirDomainCoreDump(virDomainPtr domain, const char *to, unsigned int flags){ virConnectPtr conn;
 VIR_DOMAIN_DEBUG(domain, "to=%s, flags=0x%x", to, flags);
 virResetLastError();
 virCheckDomainReturn(domain, -1); conn = domain->conn;... if (conn->driver->domainCoreDump) { int ret; char *absolute_to;
 /* We must absolutize the file path as the save is done out of process */ if (virFileAbsPath(to, &absolute_to) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("could not build absolute core file path")); goto error; }
 ret = conn->driver->domainCoreDump(domain, absolute_to, flags);
 VIR_FREE(absolute_to);
 if (ret < 0) goto error; return ret; }...}

其中conn->driver->domainCoreDump是具体到对应驱动的函数指针，这里指的是qemu驱动。函数位置在src/qemu/qemu_driver.c

 static virHypervisorDriver qemuHypervisorDriver = {    ... .domainCoreDump = qemuDomainCoreDump, /* 0.7.0 */ .domainCoreDumpWithFormat = qemuDomainCoreDumpWithFormat, /* 1.2.3 */ ...}

那接下来就是重点看qemuDomainCoreDump函数了

static intqemuDomainCoreDump(virDomainPtr dom, const char *path, unsigned int flags){ return qemuDomainCoreDumpWithFormat(dom, path,  VIR_DOMAIN_CORE_DUMP_FORMAT_RAW, flags);}

继续看qemuDomainCoreDumpWithFormat函数

static intqemuDomainCoreDumpWithFormat(virDomainPtr dom, const char *path, unsigned int dumpformat, unsigned int flags){ virQEMUDriverPtr driver = dom->conn->privateData; virDomainObjPtr vm; qemuDomainObjPrivatePtr priv = NULL; bool resume = false, paused = false; int ret = -1; virObjectEventPtr event = NULL;
 virCheckFlags(VIR_DUMP_LIVE | VIR_DUMP_CRASH | VIR_DUMP_BYPASS_CACHE | VIR_DUMP_RESET | VIR_DUMP_MEMORY_ONLY, -1);
 if (!(vm = qemuDomainObjFromDomain(dom))) return -1;... if ((ret = doCoreDump(driver, vm, path, flags, dumpformat)) < 0) goto endjob;
 paused = true;...}

qemuDomainCoreDumpWithFormat函数主要是获取domain指针，然后检查ACL策略，判断是alive模式dump还是非alive模式dump，如果是非alive的话需要停止CPU操作。接着执行doCoreDump函数

static intdoCoreDump(virQEMUDriverPtr driver, virDomainObjPtr vm, const char *path, unsigned int dump_flags, unsigned int dumpformat){ int fd = -1; int ret = -1; int rc = -1; virFileWrapperFdPtr wrapperFd = NULL; int directFlag = 0; unsigned int flags = VIR_FILE_WRAPPER_NON_BLOCKING; const char *memory_dump_format = NULL; g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); g_autoptr(virCommand) compressor = NULL;...
 if (dump_flags & VIR_DUMP_MEMORY_ONLY) { if (!(memory_dump_format = qemuDumpFormatTypeToString(dumpformat))) { virReportError(VIR_ERR_INVALID_ARG, _("unknown dumpformat '%d'"), dumpformat); goto cleanup; }
 /* qemu dumps in "elf" without dumpformat set */ if (STREQ(memory_dump_format, "elf")) memory_dump_format = NULL;
 rc = qemuDumpToFd(driver, vm, fd, QEMU_ASYNC_JOB_DUMP, memory_dump_format); } else { if (dumpformat != VIR_DOMAIN_CORE_DUMP_FORMAT_RAW) { virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", _("kdump-compressed format is only supported with " "memory-only dump")); goto cleanup; }
 if (!qemuMigrationSrcIsAllowed(driver, vm, false, 0)) goto cleanup;
 rc = qemuMigrationSrcToFile(driver, vm, fd, compressor, QEMU_ASYNC_JOB_DUMP); }...}

doCoreDump函数主要是判断flags类型，本文以VIR_DUMP_MEMORY_ONLY为例。然后打开存储dump的文件，执行qemuDumpToFd执行将内存dump至具体的文件里。

static intqemuDumpToFd(virQEMUDriverPtr driver, virDomainObjPtr vm, int fd, qemuDomainAsyncJob asyncJob, const char *dumpformat){ qemuDomainObjPrivatePtr priv = vm->privateData; bool detach = false; int ret = -1;
 if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DUMP_GUEST_MEMORY)) { virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", _("dump-guest-memory is not supported")); return -1; }... if (dumpformat) { ret = qemuMonitorGetDumpGuestMemoryCapability(priv->mon, dumpformat);
 if (ret <= 0) { virReportError(VIR_ERR_INVALID_ARG, _("unsupported dumpformat '%s' " "for this QEMU binary"), dumpformat); ignore_value(qemuDomainObjExitMonitor(driver, vm)); return -1; } }
 ret = qemuMonitorDumpToFd(priv->mon, fd, dumpformat, detach);
 if ((qemuDomainObjExitMonitor(driver, vm) < 0) || ret < 0) return -1;
 if (detach) ret = qemuDumpWaitForCompletion(vm);
 return ret;}

intqemuMonitorDumpToFd(qemuMonitorPtr mon, int fd,  const char *dumpformat, bool detach){  int ret; VIR_DEBUG("fd=%d dumpformat=%s", fd, dumpformat);
 QEMU_CHECK_MONITOR(mon);
 if (qemuMonitorSendFileHandle(mon, "dump", fd) < 0) return -1;
 ret = qemuMonitorJSONDump(mon, "fd:dump", dumpformat, detach);  if (ret < 0) { if (qemuMonitorCloseFileHandle(mon, "dump") < 0) VIR_WARN("failed to close dumping handle"); }
 return ret;}