3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1983, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2016 Joyent, Inc.
24 */
25 /*
26 * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
27 */
28
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <stdarg.h>
32 #include <unistd.h>
33 #include <fcntl.h>
34 #include <errno.h>
35 #include <string.h>
36 #include <deflt.h>
37 #include <time.h>
38 #include <syslog.h>
39 #include <stropts.h>
40 #include <pthread.h>
41 #include <limits.h>
42 #include <atomic.h>
43 #include <libnvpair.h>
75 static boolean_t have_dumpfile = B_TRUE; /* dumpfile existence */
76 static dumphdr_t corehdr, dumphdr; /* initial and terminal dumphdrs */
77 static boolean_t dump_incomplete; /* dumphdr indicates incomplete */
78 static boolean_t fm_panic; /* dump is the result of fm_panic */
79 static offset_t endoff; /* offset of end-of-dump header */
80 static int verbose; /* chatty mode */
81 static int disregard_valid_flag; /* disregard valid flag */
82 static int livedump; /* dump the current running system */
83 static int interactive; /* user invoked; no syslog */
84 static int csave; /* save dump compressed */
85 static int filemode; /* processing file, not dump device */
86 static int percent_done; /* progress indicator */
87 static int sec_done; /* progress last report time */
88 static hrtime_t startts; /* timestamp at start */
89 static volatile uint64_t saved; /* count of pages written */
90 static volatile uint64_t zpages; /* count of zero pages not written */
91 static dumpdatahdr_t datahdr; /* compression info */
92 static long coreblksize; /* preferred write size (st_blksize) */
93 static int cflag; /* run as savecore -c */
94 static int mflag; /* run as savecore -m */
95
96 /*
97 * Payload information for the events we raise. These are used
98 * in raise_event to determine what payload to include.
99 */
100 #define SC_PAYLOAD_SAVEDIR 0x0001 /* Include savedir in event */
101 #define SC_PAYLOAD_INSTANCE 0x0002 /* Include bounds instance number */
102 #define SC_PAYLOAD_IMAGEUUID 0x0004 /* Include dump OS instance uuid */
103 #define SC_PAYLOAD_CRASHTIME 0x0008 /* Include epoch crashtime */
104 #define SC_PAYLOAD_PANICSTR 0x0010 /* Include panic string */
105 #define SC_PAYLOAD_PANICSTACK 0x0020 /* Include panic string */
106 #define SC_PAYLOAD_FAILREASON 0x0040 /* Include failure reason */
107 #define SC_PAYLOAD_DUMPCOMPLETE 0x0080 /* Include completeness indicator */
108 #define SC_PAYLOAD_ISCOMPRESSED 0x0100 /* Dump is in vmdump.N form */
109 #define SC_PAYLOAD_DUMPADM_EN 0x0200 /* Is dumpadm enabled or not? */
110 #define SC_PAYLOAD_FM_PANIC 0x0400 /* Panic initiated by FMA */
111 #define SC_PAYLOAD_JUSTCHECKING 0x0800 /* Run with -c flag? */
112
113 enum sc_event_type {
114 SC_EVENT_DUMP_PENDING,
147 {
148 "savecore_failure",
149 _SC_PAYLOAD_CMN | SC_PAYLOAD_INSTANCE | SC_PAYLOAD_FAILREASON
150 },
151
152 /*
153 * SC_EVENT_DUMP_AVAILABLE
154 */
155 {
156 "dump_available",
157 _SC_PAYLOAD_CMN | SC_PAYLOAD_INSTANCE | SC_PAYLOAD_ISCOMPRESSED
158 },
159 };
160
161 static void raise_event(enum sc_event_type, char *);
162
163 static void
164 usage(void)
165 {
166 (void) fprintf(stderr,
167 "usage: %s [-Lvd] [-f dumpfile] [dirname]\n", progname);
168 exit(1);
169 }
170
171 #define SC_SL_NONE 0x0001 /* no syslog */
172 #define SC_SL_ERR 0x0002 /* syslog if !interactive, LOG_ERR */
173 #define SC_SL_WARN 0x0004 /* syslog if !interactive, LOG_WARNING */
174 #define SC_IF_VERBOSE 0x0008 /* message only if -v */
175 #define SC_IF_ISATTY 0x0010 /* message only if interactive */
176 #define SC_EXIT_OK 0x0020 /* exit(0) */
177 #define SC_EXIT_ERR 0x0040 /* exit(1) */
178 #define SC_EXIT_PEND 0x0080 /* exit(2) */
179 #define SC_EXIT_FM 0x0100 /* exit(3) */
180
181 #define _SC_ALLEXIT (SC_EXIT_OK | SC_EXIT_ERR | SC_EXIT_PEND | SC_EXIT_FM)
182
183 static void
184 logprint(uint32_t flags, char *message, ...)
185 {
186 va_list args;
187 char buf[1024];
212 break;
213 }
214 }
215 va_end(args);
216 }
217
218 switch (flags & _SC_ALLEXIT) {
219 case 0:
220 return;
221
222 case SC_EXIT_OK:
223 code = 0;
224 break;
225
226 case SC_EXIT_PEND:
227 /*
228 * Raise an ireport saying why we are exiting. Do not
229 * raise if run as savecore -m. If something in the
230 * raise_event codepath calls logprint avoid recursion.
231 */
232 if (!mflag && logprint_raised++ == 0)
233 raise_event(SC_EVENT_SAVECORE_FAILURE, buf);
234 code = 2;
235 break;
236
237 case SC_EXIT_FM:
238 code = 3;
239 break;
240
241 case SC_EXIT_ERR:
242 default:
243 if (!mflag && logprint_raised++ == 0 && have_dumpfile)
244 raise_event(SC_EVENT_SAVECORE_FAILURE, buf);
245 code = 1;
246 break;
247 }
248
249 exit(code);
250 }
251
252 /*
253 * System call / libc wrappers that exit on error.
254 */
255 static int
256 Open(const char *name, int oflags, mode_t mode)
257 {
258 int fd;
259
260 if ((fd = open64(name, oflags, mode)) == -1)
261 logprint(SC_SL_ERR | SC_EXIT_ERR, "open(\"%s\"): %s",
262 name, strerror(errno));
263 return (fd);
338 strerror(errno));
339 return (buf);
340 }
341
342 static long
343 read_number_from_file(const char *filename, long default_value)
344 {
345 long file_value = -1;
346 FILE *fp;
347
348 if ((fp = fopen(filename, "r")) != NULL) {
349 (void) fscanf(fp, "%ld", &file_value);
350 (void) fclose(fp);
351 }
352 return (file_value < 0 ? default_value : file_value);
353 }
354
355 static void
356 read_dumphdr(void)
357 {
358 if (filemode)
359 dumpfd = Open(dumpfile, O_RDONLY, 0644);
360 else
361 dumpfd = Open(dumpfile, O_RDWR | O_DSYNC, 0644);
362 endoff = llseek(dumpfd, -DUMP_OFFSET, SEEK_END) & -DUMP_OFFSET;
363 Pread(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
364 Pread(dumpfd, &datahdr, sizeof (datahdr), endoff + sizeof (dumphdr));
365
366 pagesize = dumphdr.dump_pagesize;
367
368 if (dumphdr.dump_magic != DUMP_MAGIC)
369 logprint(SC_SL_NONE | SC_EXIT_PEND, "bad magic number %x",
370 dumphdr.dump_magic);
371
372 if ((dumphdr.dump_flags & DF_VALID) == 0 && !disregard_valid_flag)
373 logprint(SC_SL_NONE | SC_IF_VERBOSE | SC_EXIT_OK,
374 "dump already processed");
375
376 if (dumphdr.dump_version != DUMP_VERSION)
377 logprint(SC_SL_NONE | SC_IF_VERBOSE | SC_EXIT_PEND,
378 "dump version (%d) != %s version (%d)",
391 DUMP_DATAHDR_VERSION);
392 } else {
393 (void) memset(&datahdr, 0, sizeof (datahdr));
394 datahdr.dump_maxcsize = pagesize;
395 }
396
397 /*
398 * Read the initial header, clear the valid bits, and compare headers.
399 * The main header may have been overwritten by swapping if we're
400 * using a swap partition as the dump device, in which case we bail.
401 */
402 Pread(dumpfd, &corehdr, sizeof (dumphdr_t), dumphdr.dump_start);
403
404 corehdr.dump_flags &= ~DF_VALID;
405 dumphdr.dump_flags &= ~DF_VALID;
406
407 if (memcmp(&corehdr, &dumphdr, sizeof (dumphdr_t)) != 0) {
408 /*
409 * Clear valid bit so we don't complain on every invocation.
410 */
411 if (!filemode)
412 Pwrite(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
413 logprint(SC_SL_ERR | SC_EXIT_ERR,
414 "initial dump header corrupt");
415 }
416 }
417
418 static void
419 check_space(int csave)
420 {
421 struct statvfs fsb;
422 int64_t spacefree, dumpsize, minfree, datasize;
423
424 if (statvfs(".", &fsb) < 0)
425 logprint(SC_SL_ERR | SC_EXIT_ERR, "statvfs: %s",
426 strerror(errno));
427
428 dumpsize = dumphdr.dump_data - dumphdr.dump_start;
429 datasize = dumphdr.dump_npages * pagesize;
430 if (!csave)
431 dumpsize += datasize;
643 }
644
645 Pwrite(corefd, &corehdr, sizeof (corehdr), coreoff);
646 coreoff += sizeof (corehdr);
647
648 Pwrite(corefd, &datahdr, sizeof (datahdr), coreoff);
649 coreoff += sizeof (datahdr);
650
651 nb = DUMP_OFFSET - (coreoff & (DUMP_OFFSET - 1));
652 if (nb > 0) {
653 Pwrite(corefd, inbuf, nb, coreoff);
654 }
655
656 free(inbuf);
657 Pwrite(corefd, &corehdr, sizeof (corehdr), corehdr.dump_start);
658
659 /*
660 * Write out the modified dump header to the dump device.
661 * The dump device has been processed, so DF_VALID is clear.
662 */
663 if (!filemode)
664 Pwrite(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
665
666 (void) close(corefd);
667 }
668
669 /*
670 * compressed streams
671 */
672 typedef struct blockhdr blockhdr_t;
673 typedef struct block block_t;
674
675 struct blockhdr {
676 block_t *head;
677 block_t *tail;
678 };
679
680 struct block {
681 block_t *next;
682 char *block;
683 int size;
1405
1406 /*
1407 * Decompress the pages
1408 */
1409 decompress_pages(corefd);
1410 (void) printf(": %ld of %ld pages saved\n", (pgcnt_t)saved,
1411 dumphdr.dump_npages);
1412
1413 if (verbose)
1414 (void) printf("%ld (%ld%%) zero pages were not written\n",
1415 (pgcnt_t)zpages, (pgcnt_t)zpages * 100 /
1416 dumphdr.dump_npages);
1417
1418 if (saved != dumphdr.dump_npages)
1419 logprint(SC_SL_WARN, "bad data after page %ld", saved);
1420
1421 /*
1422 * Write out the modified dump headers.
1423 */
1424 Pwrite(corefd, &corehdr, sizeof (corehdr), 0);
1425 if (!filemode)
1426 Pwrite(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
1427
1428 (void) close(corefd);
1429 }
1430
1431 /*
1432 * When the system panics, the kernel saves all undelivered messages (messages
1433 * that never made it out to syslogd(1M)) in the dump. At a mimimum, the
1434 * panic message itself will always fall into this category. Upon reboot,
1435 * the syslog startup script runs savecore -m to recover these messages.
1436 *
1437 * To do this, we read the unsent messages from the dump and send them to
1438 * /dev/conslog on priority band 1. This has the effect of prepending them
1439 * to any already-accumulated messages in the console backlog, thus preserving
1440 * temporal ordering across the reboot.
1441 *
1442 * Note: since savecore -m is used *only* for this purpose, it does *not*
1443 * attempt to save the crash dump. The dump will be saved later, after
1444 * syslogd(1M) starts, by the savecore startup script.
1445 */
1514
1515 if (p == NULL || strncmp(p, "vmdump", 6) != 0)
1516 p = strstr(f, "vmdump");
1517
1518 if (p != NULL && *p == '/')
1519 p++;
1520
1521 (void) sscanf(p ? p : f, "vmdump.%ld", &b);
1522
1523 return (b);
1524 }
1525
1526 static void
1527 stack_retrieve(char *stack)
1528 {
1529 summary_dump_t sd;
1530 offset_t dumpoff = -(DUMP_OFFSET + DUMP_LOGSIZE +
1531 DUMP_ERPTSIZE);
1532 dumpoff -= DUMP_SUMMARYSIZE;
1533
1534 dumpfd = Open(dumpfile, O_RDWR | O_DSYNC, 0644);
1535 dumpoff = llseek(dumpfd, dumpoff, SEEK_END) & -DUMP_OFFSET;
1536
1537 Pread(dumpfd, &sd, sizeof (summary_dump_t), dumpoff);
1538 dumpoff += sizeof (summary_dump_t);
1539
1540 if (sd.sd_magic == 0) {
1541 *stack = '\0';
1542 return;
1543 }
1544
1545 if (sd.sd_magic != SUMMARY_MAGIC) {
1546 *stack = '\0';
1547 logprint(SC_SL_NONE | SC_IF_VERBOSE,
1548 "bad summary magic %x", sd.sd_magic);
1549 return;
1550 }
1551 Pread(dumpfd, stack, STACK_BUF_SIZE, dumpoff);
1552 if (sd.sd_ssum != checksum32(stack, STACK_BUF_SIZE))
1553 logprint(SC_SL_NONE | SC_IF_VERBOSE, "bad stack checksum");
1651 int i, c, bfd;
1652 Stat_t st;
1653 struct rlimit rl;
1654 long filebounds = -1;
1655 char namelist[30], corefile[30], boundstr[30];
1656 dumpfile = NULL;
1657
1658 startts = gethrtime();
1659
1660 (void) getrlimit(RLIMIT_NOFILE, &rl);
1661 rl.rlim_cur = rl.rlim_max;
1662 (void) setrlimit(RLIMIT_NOFILE, &rl);
1663
1664 openlog(progname, LOG_ODELAY, LOG_AUTH);
1665
1666 (void) defopen("/etc/dumpadm.conf");
1667 savedir = defread("DUMPADM_SAVDIR=");
1668 if (savedir != NULL)
1669 savedir = strdup(savedir);
1670
1671 while ((c = getopt(argc, argv, "Lvcdmf:")) != EOF) {
1672 switch (c) {
1673 case 'L':
1674 livedump++;
1675 break;
1676 case 'v':
1677 verbose++;
1678 break;
1679 case 'c':
1680 cflag++;
1681 break;
1682 case 'd':
1683 disregard_valid_flag++;
1684 break;
1685 case 'm':
1686 mflag++;
1687 break;
1688 case 'f':
1689 dumpfile = optarg;
1690 filebounds = getbounds(dumpfile);
1691 break;
1692 case '?':
1693 usage();
1694 }
1695 }
1696
1697 /*
1698 * If doing something other than extracting an existing dump (i.e.
1699 * dumpfile has been provided as an option), the user must be root.
1700 */
1701 if (geteuid() != 0 && dumpfile == NULL) {
1702 (void) fprintf(stderr, "%s: %s %s\n", progname,
1703 gettext("you must be root to use"), progname);
1704 exit(1);
1705 }
1706
1707 interactive = isatty(STDOUT_FILENO);
1708
1709 if (cflag && livedump)
1710 usage();
1711
1712 if (dumpfile == NULL || livedump)
1713 dumpfd = Open("/dev/dump", O_RDONLY, 0444);
1714
1715 if (dumpfile == NULL) {
1716 dumpfile = Zalloc(MAXPATHLEN);
1717 if (ioctl(dumpfd, DIOCGETDEV, dumpfile) == -1) {
1718 have_dumpfile = B_FALSE;
1719 logprint(SC_SL_NONE | SC_IF_ISATTY | SC_EXIT_ERR,
1720 "no dump device configured");
1721 }
1722 }
1723
1724 if (mflag)
1725 return (message_save());
1726
1727 if (optind == argc - 1)
1728 savedir = argv[optind];
1729
1730 if (savedir == NULL || optind < argc - 1)
1731 usage();
1735 "dedicated dump device required");
1736
1737 (void) close(dumpfd);
1738 dumpfd = -1;
1739
1740 Stat(dumpfile, &st);
1741
1742 filemode = S_ISREG(st.st_mode);
1743
1744 if (!filemode && defread("DUMPADM_CSAVE=off") == NULL)
1745 csave = 1;
1746
1747 read_dumphdr();
1748
1749 /*
1750 * We want this message to go to the log file, but not the console.
1751 * There's no good way to do that with the existing syslog facility.
1752 * We could extend it to handle this, but there doesn't seem to be
1753 * a general need for it, so we isolate the complexity here instead.
1754 */
1755 if (dumphdr.dump_panicstring[0] != '\0') {
1756 int logfd = Open("/dev/conslog", O_WRONLY, 0644);
1757 log_ctl_t lc;
1758 struct strbuf ctl, dat;
1759 char msg[DUMP_PANICSIZE + 100];
1760 char fmt[] = "reboot after panic: %s";
1761 uint32_t msgid;
1762
1763 STRLOG_MAKE_MSGID(fmt, msgid);
1764
1765 /* LINTED: E_SEC_SPRINTF_UNBOUNDED_COPY */
1766 (void) sprintf(msg, "%s: [ID %u FACILITY_AND_PRIORITY] ",
1767 progname, msgid);
1768 /* LINTED: E_SEC_PRINTF_VAR_FMT */
1769 (void) sprintf(msg + strlen(msg), fmt,
1770 dumphdr.dump_panicstring);
1771
1772 lc.pri = LOG_AUTH | LOG_ERR;
1773 lc.flags = SL_CONSOLE | SL_LOGONLY;
1774 lc.level = 0;
1775
1784 }
1785
1786 if ((dumphdr.dump_flags & DF_COMPLETE) == 0) {
1787 logprint(SC_SL_WARN, "incomplete dump on dump device");
1788 dump_incomplete = B_TRUE;
1789 }
1790
1791 if (dumphdr.dump_fm_panic)
1792 fm_panic = B_TRUE;
1793
1794 /*
1795 * We have a valid dump on a dump device and know as much about
1796 * it as we're going to at this stage. Raise an event for
1797 * logging and so that FMA can open a case for this panic.
1798 * Avoid this step for FMA-initiated panics - FMA will replay
1799 * ereports off the dump device independently of savecore and
1800 * will make a diagnosis, so we don't want to open two cases
1801 * for the same event. Also avoid raising an event for a
1802 * livedump, or when we inflating a compressed dump.
1803 */
1804 if (!fm_panic && !livedump && !filemode)
1805 raise_event(SC_EVENT_DUMP_PENDING, NULL);
1806
1807 logprint(SC_SL_WARN, "System dump time: %s",
1808 ctime(&dumphdr.dump_crashtime));
1809
1810 /*
1811 * Option -c is designed for use from svc-dumpadm where we know
1812 * that dumpadm -n is in effect but run savecore -c just to
1813 * get the above dump_pending_on_device event raised. If it is run
1814 * interactively then just print further panic details.
1815 */
1816 if (cflag) {
1817 char *disabled = defread("DUMPADM_ENABLE=no");
1818 int lvl = interactive ? SC_SL_WARN : SC_SL_ERR;
1819 int ec = fm_panic ? SC_EXIT_FM : SC_EXIT_PEND;
1820
1821 logprint(lvl | ec,
1822 "Panic crashdump pending on dump device%s "
1823 "run savecore(1M) manually to extract. "
1824 "Image UUID %s%s.",
1840 bounds = filebounds;
1841
1842 if (csave) {
1843 size_t metrics_size = datahdr.dump_metrics;
1844
1845 (void) sprintf(corefile, "vmdump.%ld", bounds);
1846
1847 datahdr.dump_metrics = 0;
1848
1849 logprint(SC_SL_ERR,
1850 "Saving compressed system crash dump in %s/%s",
1851 savedir, corefile);
1852
1853 copy_crashfile(corefile);
1854
1855 /*
1856 * Raise a fault management event that indicates the system
1857 * has panicked. We know a reasonable amount about the
1858 * condition at this time, but the dump is still compressed.
1859 */
1860 if (!livedump && !fm_panic)
1861 raise_event(SC_EVENT_DUMP_AVAILABLE, NULL);
1862
1863 if (metrics_size > 0) {
1864 int sec = (gethrtime() - startts) / 1000 / 1000 / 1000;
1865 FILE *mfile = fopen(METRICSFILE, "a");
1866 char *metrics = Zalloc(metrics_size + 1);
1867
1868 Pread(dumpfd, metrics, metrics_size, endoff +
1869 sizeof (dumphdr) + sizeof (datahdr));
1870
1871 if (sec < 1)
1872 sec = 1;
1873
1874 if (mfile == NULL) {
1875 logprint(SC_SL_WARN,
1876 "Can't create %s:\n%s",
1877 METRICSFILE, metrics);
1878 } else {
1879 (void) fprintf(mfile, "[[[[,,,");
1880 for (i = 0; i < argc; i++)
1909 "\n'savecore -vf %s/%s'",
1910 savedir, corefile);
1911
1912 } else {
1913 (void) sprintf(namelist, "unix.%ld", bounds);
1914 (void) sprintf(corefile, "vmcore.%ld", bounds);
1915
1916 if (interactive && filebounds >= 0 && access(corefile, F_OK)
1917 == 0)
1918 logprint(SC_SL_NONE | SC_EXIT_ERR,
1919 "%s already exists: remove with "
1920 "'rm -f %s/{unix,vmcore}.%ld'",
1921 corefile, savedir, bounds);
1922
1923 logprint(SC_SL_ERR,
1924 "saving system crash dump in %s/{unix,vmcore}.%ld",
1925 savedir, bounds);
1926
1927 build_corefile(namelist, corefile);
1928
1929 if (!livedump && !filemode && !fm_panic)
1930 raise_event(SC_EVENT_DUMP_AVAILABLE, NULL);
1931
1932 if (access(METRICSFILE, F_OK) == 0) {
1933 int sec = (gethrtime() - startts) / 1000 / 1000 / 1000;
1934 FILE *mfile = fopen(METRICSFILE, "a");
1935
1936 if (sec < 1)
1937 sec = 1;
1938
1939 if (mfile == NULL) {
1940 logprint(SC_SL_WARN,
1941 "Can't create %s: %s",
1942 METRICSFILE, strerror(errno));
1943 } else {
1944 (void) fprintf(mfile, "[[[[,,,");
1945 for (i = 0; i < argc; i++)
1946 (void) fprintf(mfile, "%s ", argv[i]);
1947 (void) fprintf(mfile, "\n");
1948 (void) fprintf(mfile, ",,,%s/%s\n", savedir,
1949 corefile);
|
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1983, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2019 Joyent, Inc.
24 */
25 /*
26 * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
27 */
28
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <stdarg.h>
32 #include <unistd.h>
33 #include <fcntl.h>
34 #include <errno.h>
35 #include <string.h>
36 #include <deflt.h>
37 #include <time.h>
38 #include <syslog.h>
39 #include <stropts.h>
40 #include <pthread.h>
41 #include <limits.h>
42 #include <atomic.h>
43 #include <libnvpair.h>
75 static boolean_t have_dumpfile = B_TRUE; /* dumpfile existence */
76 static dumphdr_t corehdr, dumphdr; /* initial and terminal dumphdrs */
77 static boolean_t dump_incomplete; /* dumphdr indicates incomplete */
78 static boolean_t fm_panic; /* dump is the result of fm_panic */
79 static offset_t endoff; /* offset of end-of-dump header */
80 static int verbose; /* chatty mode */
81 static int disregard_valid_flag; /* disregard valid flag */
82 static int livedump; /* dump the current running system */
83 static int interactive; /* user invoked; no syslog */
84 static int csave; /* save dump compressed */
85 static int filemode; /* processing file, not dump device */
86 static int percent_done; /* progress indicator */
87 static int sec_done; /* progress last report time */
88 static hrtime_t startts; /* timestamp at start */
89 static volatile uint64_t saved; /* count of pages written */
90 static volatile uint64_t zpages; /* count of zero pages not written */
91 static dumpdatahdr_t datahdr; /* compression info */
92 static long coreblksize; /* preferred write size (st_blksize) */
93 static int cflag; /* run as savecore -c */
94 static int mflag; /* run as savecore -m */
95 static int rflag; /* run as savecore -r */
96
97 /*
98 * Payload information for the events we raise. These are used
99 * in raise_event to determine what payload to include.
100 */
101 #define SC_PAYLOAD_SAVEDIR 0x0001 /* Include savedir in event */
102 #define SC_PAYLOAD_INSTANCE 0x0002 /* Include bounds instance number */
103 #define SC_PAYLOAD_IMAGEUUID 0x0004 /* Include dump OS instance uuid */
104 #define SC_PAYLOAD_CRASHTIME 0x0008 /* Include epoch crashtime */
105 #define SC_PAYLOAD_PANICSTR 0x0010 /* Include panic string */
106 #define SC_PAYLOAD_PANICSTACK 0x0020 /* Include panic string */
107 #define SC_PAYLOAD_FAILREASON 0x0040 /* Include failure reason */
108 #define SC_PAYLOAD_DUMPCOMPLETE 0x0080 /* Include completeness indicator */
109 #define SC_PAYLOAD_ISCOMPRESSED 0x0100 /* Dump is in vmdump.N form */
110 #define SC_PAYLOAD_DUMPADM_EN 0x0200 /* Is dumpadm enabled or not? */
111 #define SC_PAYLOAD_FM_PANIC 0x0400 /* Panic initiated by FMA */
112 #define SC_PAYLOAD_JUSTCHECKING 0x0800 /* Run with -c flag? */
113
114 enum sc_event_type {
115 SC_EVENT_DUMP_PENDING,
148 {
149 "savecore_failure",
150 _SC_PAYLOAD_CMN | SC_PAYLOAD_INSTANCE | SC_PAYLOAD_FAILREASON
151 },
152
153 /*
154 * SC_EVENT_DUMP_AVAILABLE
155 */
156 {
157 "dump_available",
158 _SC_PAYLOAD_CMN | SC_PAYLOAD_INSTANCE | SC_PAYLOAD_ISCOMPRESSED
159 },
160 };
161
162 static void raise_event(enum sc_event_type, char *);
163
164 static void
165 usage(void)
166 {
167 (void) fprintf(stderr,
168 "usage: %s [-L | -r] [-vd] [-f dumpfile] [dirname]\n", progname);
169 exit(1);
170 }
171
172 #define SC_SL_NONE 0x0001 /* no syslog */
173 #define SC_SL_ERR 0x0002 /* syslog if !interactive, LOG_ERR */
174 #define SC_SL_WARN 0x0004 /* syslog if !interactive, LOG_WARNING */
175 #define SC_IF_VERBOSE 0x0008 /* message only if -v */
176 #define SC_IF_ISATTY 0x0010 /* message only if interactive */
177 #define SC_EXIT_OK 0x0020 /* exit(0) */
178 #define SC_EXIT_ERR 0x0040 /* exit(1) */
179 #define SC_EXIT_PEND 0x0080 /* exit(2) */
180 #define SC_EXIT_FM 0x0100 /* exit(3) */
181
182 #define _SC_ALLEXIT (SC_EXIT_OK | SC_EXIT_ERR | SC_EXIT_PEND | SC_EXIT_FM)
183
184 static void
185 logprint(uint32_t flags, char *message, ...)
186 {
187 va_list args;
188 char buf[1024];
213 break;
214 }
215 }
216 va_end(args);
217 }
218
219 switch (flags & _SC_ALLEXIT) {
220 case 0:
221 return;
222
223 case SC_EXIT_OK:
224 code = 0;
225 break;
226
227 case SC_EXIT_PEND:
228 /*
229 * Raise an ireport saying why we are exiting. Do not
230 * raise if run as savecore -m. If something in the
231 * raise_event codepath calls logprint avoid recursion.
232 */
233 if (!mflag && !rflag && logprint_raised++ == 0)
234 raise_event(SC_EVENT_SAVECORE_FAILURE, buf);
235 code = 2;
236 break;
237
238 case SC_EXIT_FM:
239 code = 3;
240 break;
241
242 case SC_EXIT_ERR:
243 default:
244 if (!mflag && !rflag && logprint_raised++ == 0 && have_dumpfile)
245 raise_event(SC_EVENT_SAVECORE_FAILURE, buf);
246 code = 1;
247 break;
248 }
249
250 exit(code);
251 }
252
253 /*
254 * System call / libc wrappers that exit on error.
255 */
256 static int
257 Open(const char *name, int oflags, mode_t mode)
258 {
259 int fd;
260
261 if ((fd = open64(name, oflags, mode)) == -1)
262 logprint(SC_SL_ERR | SC_EXIT_ERR, "open(\"%s\"): %s",
263 name, strerror(errno));
264 return (fd);
339 strerror(errno));
340 return (buf);
341 }
342
343 static long
344 read_number_from_file(const char *filename, long default_value)
345 {
346 long file_value = -1;
347 FILE *fp;
348
349 if ((fp = fopen(filename, "r")) != NULL) {
350 (void) fscanf(fp, "%ld", &file_value);
351 (void) fclose(fp);
352 }
353 return (file_value < 0 ? default_value : file_value);
354 }
355
356 static void
357 read_dumphdr(void)
358 {
359 if (filemode || rflag)
360 dumpfd = Open(dumpfile, O_RDONLY, 0644);
361 else
362 dumpfd = Open(dumpfile, O_RDWR | O_DSYNC, 0644);
363 endoff = llseek(dumpfd, -DUMP_OFFSET, SEEK_END) & -DUMP_OFFSET;
364 Pread(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
365 Pread(dumpfd, &datahdr, sizeof (datahdr), endoff + sizeof (dumphdr));
366
367 pagesize = dumphdr.dump_pagesize;
368
369 if (dumphdr.dump_magic != DUMP_MAGIC)
370 logprint(SC_SL_NONE | SC_EXIT_PEND, "bad magic number %x",
371 dumphdr.dump_magic);
372
373 if ((dumphdr.dump_flags & DF_VALID) == 0 && !disregard_valid_flag)
374 logprint(SC_SL_NONE | SC_IF_VERBOSE | SC_EXIT_OK,
375 "dump already processed");
376
377 if (dumphdr.dump_version != DUMP_VERSION)
378 logprint(SC_SL_NONE | SC_IF_VERBOSE | SC_EXIT_PEND,
379 "dump version (%d) != %s version (%d)",
392 DUMP_DATAHDR_VERSION);
393 } else {
394 (void) memset(&datahdr, 0, sizeof (datahdr));
395 datahdr.dump_maxcsize = pagesize;
396 }
397
398 /*
399 * Read the initial header, clear the valid bits, and compare headers.
400 * The main header may have been overwritten by swapping if we're
401 * using a swap partition as the dump device, in which case we bail.
402 */
403 Pread(dumpfd, &corehdr, sizeof (dumphdr_t), dumphdr.dump_start);
404
405 corehdr.dump_flags &= ~DF_VALID;
406 dumphdr.dump_flags &= ~DF_VALID;
407
408 if (memcmp(&corehdr, &dumphdr, sizeof (dumphdr_t)) != 0) {
409 /*
410 * Clear valid bit so we don't complain on every invocation.
411 */
412 if (!filemode && !rflag)
413 Pwrite(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
414 logprint(SC_SL_ERR | SC_EXIT_ERR,
415 "initial dump header corrupt");
416 }
417 }
418
419 static void
420 check_space(int csave)
421 {
422 struct statvfs fsb;
423 int64_t spacefree, dumpsize, minfree, datasize;
424
425 if (statvfs(".", &fsb) < 0)
426 logprint(SC_SL_ERR | SC_EXIT_ERR, "statvfs: %s",
427 strerror(errno));
428
429 dumpsize = dumphdr.dump_data - dumphdr.dump_start;
430 datasize = dumphdr.dump_npages * pagesize;
431 if (!csave)
432 dumpsize += datasize;
644 }
645
646 Pwrite(corefd, &corehdr, sizeof (corehdr), coreoff);
647 coreoff += sizeof (corehdr);
648
649 Pwrite(corefd, &datahdr, sizeof (datahdr), coreoff);
650 coreoff += sizeof (datahdr);
651
652 nb = DUMP_OFFSET - (coreoff & (DUMP_OFFSET - 1));
653 if (nb > 0) {
654 Pwrite(corefd, inbuf, nb, coreoff);
655 }
656
657 free(inbuf);
658 Pwrite(corefd, &corehdr, sizeof (corehdr), corehdr.dump_start);
659
660 /*
661 * Write out the modified dump header to the dump device.
662 * The dump device has been processed, so DF_VALID is clear.
663 */
664 if (!filemode && !rflag)
665 Pwrite(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
666
667 (void) close(corefd);
668 }
669
670 /*
671 * compressed streams
672 */
673 typedef struct blockhdr blockhdr_t;
674 typedef struct block block_t;
675
676 struct blockhdr {
677 block_t *head;
678 block_t *tail;
679 };
680
681 struct block {
682 block_t *next;
683 char *block;
684 int size;
1406
1407 /*
1408 * Decompress the pages
1409 */
1410 decompress_pages(corefd);
1411 (void) printf(": %ld of %ld pages saved\n", (pgcnt_t)saved,
1412 dumphdr.dump_npages);
1413
1414 if (verbose)
1415 (void) printf("%ld (%ld%%) zero pages were not written\n",
1416 (pgcnt_t)zpages, (pgcnt_t)zpages * 100 /
1417 dumphdr.dump_npages);
1418
1419 if (saved != dumphdr.dump_npages)
1420 logprint(SC_SL_WARN, "bad data after page %ld", saved);
1421
1422 /*
1423 * Write out the modified dump headers.
1424 */
1425 Pwrite(corefd, &corehdr, sizeof (corehdr), 0);
1426 if (!filemode && !rflag)
1427 Pwrite(dumpfd, &dumphdr, sizeof (dumphdr), endoff);
1428
1429 (void) close(corefd);
1430 }
1431
1432 /*
1433 * When the system panics, the kernel saves all undelivered messages (messages
1434 * that never made it out to syslogd(1M)) in the dump. At a mimimum, the
1435 * panic message itself will always fall into this category. Upon reboot,
1436 * the syslog startup script runs savecore -m to recover these messages.
1437 *
1438 * To do this, we read the unsent messages from the dump and send them to
1439 * /dev/conslog on priority band 1. This has the effect of prepending them
1440 * to any already-accumulated messages in the console backlog, thus preserving
1441 * temporal ordering across the reboot.
1442 *
1443 * Note: since savecore -m is used *only* for this purpose, it does *not*
1444 * attempt to save the crash dump. The dump will be saved later, after
1445 * syslogd(1M) starts, by the savecore startup script.
1446 */
1515
1516 if (p == NULL || strncmp(p, "vmdump", 6) != 0)
1517 p = strstr(f, "vmdump");
1518
1519 if (p != NULL && *p == '/')
1520 p++;
1521
1522 (void) sscanf(p ? p : f, "vmdump.%ld", &b);
1523
1524 return (b);
1525 }
1526
1527 static void
1528 stack_retrieve(char *stack)
1529 {
1530 summary_dump_t sd;
1531 offset_t dumpoff = -(DUMP_OFFSET + DUMP_LOGSIZE +
1532 DUMP_ERPTSIZE);
1533 dumpoff -= DUMP_SUMMARYSIZE;
1534
1535 if (rflag)
1536 dumpfd = Open(dumpfile, O_RDONLY, 0644);
1537 else
1538 dumpfd = Open(dumpfile, O_RDWR | O_DSYNC, 0644);
1539 dumpoff = llseek(dumpfd, dumpoff, SEEK_END) & -DUMP_OFFSET;
1540
1541 Pread(dumpfd, &sd, sizeof (summary_dump_t), dumpoff);
1542 dumpoff += sizeof (summary_dump_t);
1543
1544 if (sd.sd_magic == 0) {
1545 *stack = '\0';
1546 return;
1547 }
1548
1549 if (sd.sd_magic != SUMMARY_MAGIC) {
1550 *stack = '\0';
1551 logprint(SC_SL_NONE | SC_IF_VERBOSE,
1552 "bad summary magic %x", sd.sd_magic);
1553 return;
1554 }
1555 Pread(dumpfd, stack, STACK_BUF_SIZE, dumpoff);
1556 if (sd.sd_ssum != checksum32(stack, STACK_BUF_SIZE))
1557 logprint(SC_SL_NONE | SC_IF_VERBOSE, "bad stack checksum");
1655 int i, c, bfd;
1656 Stat_t st;
1657 struct rlimit rl;
1658 long filebounds = -1;
1659 char namelist[30], corefile[30], boundstr[30];
1660 dumpfile = NULL;
1661
1662 startts = gethrtime();
1663
1664 (void) getrlimit(RLIMIT_NOFILE, &rl);
1665 rl.rlim_cur = rl.rlim_max;
1666 (void) setrlimit(RLIMIT_NOFILE, &rl);
1667
1668 openlog(progname, LOG_ODELAY, LOG_AUTH);
1669
1670 (void) defopen("/etc/dumpadm.conf");
1671 savedir = defread("DUMPADM_SAVDIR=");
1672 if (savedir != NULL)
1673 savedir = strdup(savedir);
1674
1675 while ((c = getopt(argc, argv, "Lvcdmf:r")) != EOF) {
1676 switch (c) {
1677 case 'L':
1678 livedump++;
1679 break;
1680 case 'v':
1681 verbose++;
1682 break;
1683 case 'c':
1684 cflag++;
1685 break;
1686 case 'd':
1687 disregard_valid_flag++;
1688 break;
1689 case 'm':
1690 mflag++;
1691 break;
1692 case 'r':
1693 rflag++;
1694 break;
1695 case 'f':
1696 dumpfile = optarg;
1697 filebounds = getbounds(dumpfile);
1698 break;
1699 case '?':
1700 usage();
1701 }
1702 }
1703
1704 /*
1705 * If doing something other than extracting an existing dump (i.e.
1706 * dumpfile has been provided as an option), the user must be root.
1707 */
1708 if (geteuid() != 0 && dumpfile == NULL) {
1709 (void) fprintf(stderr, "%s: %s %s\n", progname,
1710 gettext("you must be root to use"), progname);
1711 exit(1);
1712 }
1713
1714 interactive = isatty(STDOUT_FILENO);
1715
1716 if (cflag && livedump)
1717 usage();
1718
1719 if (rflag && (cflag || mflag || livedump))
1720 usage();
1721
1722 if (dumpfile == NULL || livedump)
1723 dumpfd = Open("/dev/dump", O_RDONLY, 0444);
1724
1725 if (dumpfile == NULL) {
1726 dumpfile = Zalloc(MAXPATHLEN);
1727 if (ioctl(dumpfd, DIOCGETDEV, dumpfile) == -1) {
1728 have_dumpfile = B_FALSE;
1729 logprint(SC_SL_NONE | SC_IF_ISATTY | SC_EXIT_ERR,
1730 "no dump device configured");
1731 }
1732 }
1733
1734 if (mflag)
1735 return (message_save());
1736
1737 if (optind == argc - 1)
1738 savedir = argv[optind];
1739
1740 if (savedir == NULL || optind < argc - 1)
1741 usage();
1745 "dedicated dump device required");
1746
1747 (void) close(dumpfd);
1748 dumpfd = -1;
1749
1750 Stat(dumpfile, &st);
1751
1752 filemode = S_ISREG(st.st_mode);
1753
1754 if (!filemode && defread("DUMPADM_CSAVE=off") == NULL)
1755 csave = 1;
1756
1757 read_dumphdr();
1758
1759 /*
1760 * We want this message to go to the log file, but not the console.
1761 * There's no good way to do that with the existing syslog facility.
1762 * We could extend it to handle this, but there doesn't seem to be
1763 * a general need for it, so we isolate the complexity here instead.
1764 */
1765 if (dumphdr.dump_panicstring[0] != '\0' && !rflag) {
1766 int logfd = Open("/dev/conslog", O_WRONLY, 0644);
1767 log_ctl_t lc;
1768 struct strbuf ctl, dat;
1769 char msg[DUMP_PANICSIZE + 100];
1770 char fmt[] = "reboot after panic: %s";
1771 uint32_t msgid;
1772
1773 STRLOG_MAKE_MSGID(fmt, msgid);
1774
1775 /* LINTED: E_SEC_SPRINTF_UNBOUNDED_COPY */
1776 (void) sprintf(msg, "%s: [ID %u FACILITY_AND_PRIORITY] ",
1777 progname, msgid);
1778 /* LINTED: E_SEC_PRINTF_VAR_FMT */
1779 (void) sprintf(msg + strlen(msg), fmt,
1780 dumphdr.dump_panicstring);
1781
1782 lc.pri = LOG_AUTH | LOG_ERR;
1783 lc.flags = SL_CONSOLE | SL_LOGONLY;
1784 lc.level = 0;
1785
1794 }
1795
1796 if ((dumphdr.dump_flags & DF_COMPLETE) == 0) {
1797 logprint(SC_SL_WARN, "incomplete dump on dump device");
1798 dump_incomplete = B_TRUE;
1799 }
1800
1801 if (dumphdr.dump_fm_panic)
1802 fm_panic = B_TRUE;
1803
1804 /*
1805 * We have a valid dump on a dump device and know as much about
1806 * it as we're going to at this stage. Raise an event for
1807 * logging and so that FMA can open a case for this panic.
1808 * Avoid this step for FMA-initiated panics - FMA will replay
1809 * ereports off the dump device independently of savecore and
1810 * will make a diagnosis, so we don't want to open two cases
1811 * for the same event. Also avoid raising an event for a
1812 * livedump, or when we inflating a compressed dump.
1813 */
1814 if (!fm_panic && !livedump && !filemode && !rflag)
1815 raise_event(SC_EVENT_DUMP_PENDING, NULL);
1816
1817 logprint(SC_SL_WARN, "System dump time: %s",
1818 ctime(&dumphdr.dump_crashtime));
1819
1820 /*
1821 * Option -c is designed for use from svc-dumpadm where we know
1822 * that dumpadm -n is in effect but run savecore -c just to
1823 * get the above dump_pending_on_device event raised. If it is run
1824 * interactively then just print further panic details.
1825 */
1826 if (cflag) {
1827 char *disabled = defread("DUMPADM_ENABLE=no");
1828 int lvl = interactive ? SC_SL_WARN : SC_SL_ERR;
1829 int ec = fm_panic ? SC_EXIT_FM : SC_EXIT_PEND;
1830
1831 logprint(lvl | ec,
1832 "Panic crashdump pending on dump device%s "
1833 "run savecore(1M) manually to extract. "
1834 "Image UUID %s%s.",
1850 bounds = filebounds;
1851
1852 if (csave) {
1853 size_t metrics_size = datahdr.dump_metrics;
1854
1855 (void) sprintf(corefile, "vmdump.%ld", bounds);
1856
1857 datahdr.dump_metrics = 0;
1858
1859 logprint(SC_SL_ERR,
1860 "Saving compressed system crash dump in %s/%s",
1861 savedir, corefile);
1862
1863 copy_crashfile(corefile);
1864
1865 /*
1866 * Raise a fault management event that indicates the system
1867 * has panicked. We know a reasonable amount about the
1868 * condition at this time, but the dump is still compressed.
1869 */
1870 if (!livedump && !fm_panic && !rflag)
1871 raise_event(SC_EVENT_DUMP_AVAILABLE, NULL);
1872
1873 if (metrics_size > 0) {
1874 int sec = (gethrtime() - startts) / 1000 / 1000 / 1000;
1875 FILE *mfile = fopen(METRICSFILE, "a");
1876 char *metrics = Zalloc(metrics_size + 1);
1877
1878 Pread(dumpfd, metrics, metrics_size, endoff +
1879 sizeof (dumphdr) + sizeof (datahdr));
1880
1881 if (sec < 1)
1882 sec = 1;
1883
1884 if (mfile == NULL) {
1885 logprint(SC_SL_WARN,
1886 "Can't create %s:\n%s",
1887 METRICSFILE, metrics);
1888 } else {
1889 (void) fprintf(mfile, "[[[[,,,");
1890 for (i = 0; i < argc; i++)
1919 "\n'savecore -vf %s/%s'",
1920 savedir, corefile);
1921
1922 } else {
1923 (void) sprintf(namelist, "unix.%ld", bounds);
1924 (void) sprintf(corefile, "vmcore.%ld", bounds);
1925
1926 if (interactive && filebounds >= 0 && access(corefile, F_OK)
1927 == 0)
1928 logprint(SC_SL_NONE | SC_EXIT_ERR,
1929 "%s already exists: remove with "
1930 "'rm -f %s/{unix,vmcore}.%ld'",
1931 corefile, savedir, bounds);
1932
1933 logprint(SC_SL_ERR,
1934 "saving system crash dump in %s/{unix,vmcore}.%ld",
1935 savedir, bounds);
1936
1937 build_corefile(namelist, corefile);
1938
1939 if (!livedump && !filemode && !fm_panic && !rflag)
1940 raise_event(SC_EVENT_DUMP_AVAILABLE, NULL);
1941
1942 if (access(METRICSFILE, F_OK) == 0) {
1943 int sec = (gethrtime() - startts) / 1000 / 1000 / 1000;
1944 FILE *mfile = fopen(METRICSFILE, "a");
1945
1946 if (sec < 1)
1947 sec = 1;
1948
1949 if (mfile == NULL) {
1950 logprint(SC_SL_WARN,
1951 "Can't create %s: %s",
1952 METRICSFILE, strerror(errno));
1953 } else {
1954 (void) fprintf(mfile, "[[[[,,,");
1955 for (i = 0; i < argc; i++)
1956 (void) fprintf(mfile, "%s ", argv[i]);
1957 (void) fprintf(mfile, "\n");
1958 (void) fprintf(mfile, ",,,%s/%s\n", savedir,
1959 corefile);
|