oot 1456 Oct 26 04:34 secure -rw-r--r-- 1 root root 4938 Oct 26 04:34 cron [root@mdb1 log]# tail messages Oct 26 04:02:06 mdb1.prod.prv syslog-ng[1211]: Configuration reload request received, reloading configuration; Oct 26 04:02:06 mdb1.prod.prv syslog-ng[1211]: Syslog connection established; fd='11', server='AF_INET(10.240.46.127:514)', local='AF_INET(0.0.0.0:0)' Oct 26 04:07:33 mdb1.prod.prv ntpd[1258]: frequency error -512 PPM exceeds tolerance 500 PPM Oct 26 04:16:08 mdb1.prod.prv ntpd[1258]: frequency error -512 PPM exceeds tolerance 500 PPM Oct 26 04:22:06 mdb1.prod.prv -- MARK -- Oct 26 04:33:15 mdb1.prod.prv ntpd[1258]: synchronized to 10.126.38.185, stratum 2 Oct 26 04:33:15 mdb1.prod.prv ntpd[1258]: frequency error -512 PPM exceeds tolerance 500 PPM [root@mdb1 log]# date Wed Oct 26 04:34:52 EDT 2011 [root@mdb1 log]# tail cron Oct 26 04:27:01 mdb1.prod.prv crond[7784]: (root) CMD (for FILE in `ls /opt/sm/bin/gmetric-*`; do command $FILE; done >/dev/null 2>&1) Oct 26 04:28:01 mdb1.prod.prv crond[7850]: (root) CMD (for FILE in `ls /opt/sm/bin/gmetric-*`; do command $FILE; done >/dev/null 2>&1) Oct 26 04:29:02 mdb1.prod.prv crond[7906]: (root) CMD (for FILE in `ls /opt/sm/bin/gmetric-*`; do command $FILE; done >/dev/null 2>&1) Oct 26 04:30:01 mdb1.prod.prv crond[7982]: (root) CMD (/usr/lib64/sa/sa1 1 1) top - 04:36:45 up 12 days, 5:55, 1 user, load average: 0.00, 0.00, 0.04 Tasks: 270 total, 1 running, 268 sleeping, 0 stopped, 1 zombie Cpu(s): 0.2%us, 0.2%sy, 0.0%ni, 99.5%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st Mem: 15736360k total, 15710584k used, 25776k free, 22864k buffers Swap: 0k total, 0k used, 0k free, 13992740k cached PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 2809 mongod 16 0 54.9g 8.3g 7.5g S 0.0 55.2 616:05.35 mongod 5019 root 18 0 148m 56m 3668 S 0.0 0.4 0:04.73 chef-client 4812 root 16 0 44032 11m 3408 S 0.0 0.1 0:16.51 mongodump ====== Last login: Tue Oct 25 14:05:11 2011 from 10.98.15.4 ------------------------------------------------------------------------------ WARNING: To protect the system from unauthorized use and to ensure that the system is functioning properly, activities on this system are monitored and recorded and subject to audit. Use of this system is expressed consent to such monitoring and recording. Any unauthorized access or use of this Automated Information System is prohibited and could be subject to criminal and civil penalties. ------------------------------------------------------------------------------ [root@mdb1 ~]# mongo MongoDB shell version: 1.8.3 connecting to: test [root@mdb1 ~]# cd /var/log/mongo/ [root@mdb1 mongo]# ls -ltr total 4420 -rw-r----- 1 mongod mongod 0 Oct 15 04:02 mongod.log.2011-10-15T08-02-07 -rw-r----- 1 mongod mongod 371342 Oct 15 04:02 mongod.log-20111015.gz -rw-r--r-- 1 mongod mongod 0 Oct 16 04:02 mongod.log.2011-10-16T08-02-08 -rw-r--r-- 1 mongod mongod 291615 Oct 16 04:02 mongod.log-20111016.gz -rw-r--r-- 1 mongod mongod 0 Oct 17 04:02 mongod.log.2011-10-17T08-02-07 -rw-r--r-- 1 mongod mongod 274836 Oct 17 04:02 mongod.log-20111017.gz -rw-r--r-- 1 mongod mongod 0 Oct 18 04:02 mongod.log.2011-10-18T08-02-05 -rw-r--r-- 1 mongod mongod 342787 Oct 18 04:02 mongod.log-20111018.gz -rw-r--r-- 1 mongod mongod 0 Oct 19 04:02 mongod.log.2011-10-19T08-02-05 -rw-r--r-- 1 mongod mongod 372396 Oct 19 04:02 mongod.log-20111019.gz -rw-r--r-- 1 mongod mongod 0 Oct 20 04:02 mongod.log.2011-10-20T08-02-06 -rw-r--r-- 1 mongod mongod 385000 Oct 20 04:02 mongod.log-20111020.gz -rw-r--r-- 1 mongod mongod 0 Oct 21 04:02 mongod.log.2011-10-21T08-02-06 -rw-r--r-- 1 mongod mongod 391162 Oct 21 04:02 mongod.log-20111021.gz -rw-r--r-- 1 mongod mongod 0 Oct 22 04:02 mongod.log.2011-10-22T08-02-05 -rw-r--r-- 1 mongod mongod 359021 Oct 22 04:02 mongod.log-20111022.gz -rw-r--r-- 1 mongod mongod 0 Oct 23 04:02 mongod.log.2011-10-23T08-02-06 -rw-r--r-- 1 mongod mongod 360425 Oct 23 04:02 mongod.log-20111023.gz -rw-r--r-- 1 mongod mongod 0 Oct 24 04:02 mongod.log.2011-10-24T08-02-05 -rw-r--r-- 1 mongod mongod 372941 Oct 24 04:02 mongod.log-20111024.gz -rw-r--r-- 1 mongod mongod 0 Oct 25 04:02 mongod.log.2011-10-25T08-02-05 -rw-r--r-- 1 mongod mongod 444831 Oct 25 04:02 mongod.log-20111025.gz -rw-r--r-- 1 mongod mongod 0 Oct 26 04:02 mongod.log.2011-10-26T08-02-06 -rw-r--r-- 1 mongod mongod 489671 Oct 26 04:02 mongod.log-20111026.gz -rw-r--r-- 1 mongod mongod 0 Oct 26 04:02 mongod.log [root@mdb1 mongo]# zcat mongod.log-20111026.gz |tail Wed Oct 26 04:02:01 [initandlisten] connection accepted from 10.114.154.222:35793 #58385 Wed Oct 26 04:02:01 [conn58384] end connection 10.114.154.222:35792 Wed Oct 26 04:02:01 [conn58385] end connection 10.114.154.222:35793 Wed Oct 26 04:02:01 [initandlisten] connection accepted from 10.114.154.222:35796 #58386 Wed Oct 26 04:02:01 [conn58386] end connection 10.114.154.222:35796 Wed Oct 26 04:02:04 [conn32435] getmore local.oplog.rs cid:6329820952137827533 getMore: { ts: { $gte: new Date(5665527532368166946) } } bytes:20 nreturned:0 3107ms Wed Oct 26 04:02:04 [conn32436] getmore local.oplog.rs cid:1290508237166085326 getMore: { ts: { $gte: new Date(5665527532368166946) } } bytes:20 nreturned:0 3110ms Wed Oct 26 04:02:04 [dur] lsn set 1053593475 Wed Oct 26 04:02:06 [initandlisten] connection accepted from 127.0.0.1:55325 #58387 Wed Oct 26 04:02:06 [initandlisten] connection accepted from 127.0.0.1:55325 #58387 [root@mdb1 mongo]# vmstat 1 procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------ r b swpd free buff cache si so bi bo in cs us sy id wa st 0 0 0 26708 22748 14002148 0 0 857 847 5 0 4 1 92 3 0 0 0 0 26568 22748 14002176 0 0 0 0 249 435 0 0 100 0 0 0 0 0 26568 22748 14002176 0 0 0 0 392 479 2 0 98 0 0 0 0 0 26568 22748 14002176 0 0 0 0 231 426 0 0 100 0 0 0 0 0 26568 22748 14002176 0 0 0 0 239 425 0 0 100 0 0 0 0 0 26568 22756 14002172 0 0 0 12 242 440 0 0 100 0 0 0 0 0 26568 22756 14002172 0 0 0 0 246 438 0 0 100 0 0 0 0 0 26568 22756 14002176 0 0 0 0 237 432 0 0 100 0 0 0 0 0 26568 22756 14002176 0 0 0 0 223 415 0 0 100 0 0 0 0 0 26568 22756 14002176 0 0 0 0 238 434 0 0 100 0 0 0 0 0 26568 22756 14002176 0 0 0 0 238 432 0 0 100 0 0 0 0 0 26568 22756 14002176 0 0 0 0 319 442 0 0 100 0 0 [root@mdb1 mongo]# [root@mdb1 mongo]# [root@mdb1 mongo]# iostat -x 1 Linux 2.6.21.7-2.fc8xen (mdb1.prod.prv) 10/26/2011 avg-cpu: %user %nice %system %iowait %steal %idle 4.34 0.00 0.52 3.42 0.03 91.69 Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util sda1 0.46 2.91 2.92 1.84 85.91 38.11 26.03 0.04 8.69 0.92 0.44 sdf 0.08 3.11 21.55 5.41 1332.92 229.63 57.95 0.28 10.33 0.77 2.07 sdg 0.04 62.31 39.69 48.27 3260.44 3677.68 78.88 3.53 40.17 1.13 9.96 sdh 0.04 37.07 25.68 36.21 2180.96 2834.30 81.04 0.60 9.71 1.03 6.35 avg-cpu: %user %nice %system %iowait %steal %idle 0.00 0.00 0.28 0.00 0.00 99.72 Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util sda1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdf 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdg 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdh 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 avg-cpu: %user %nice %system %iowait %steal %idle 0.00 0.00 0.00 0.00 0.00 100.00 Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util sda1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdf 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdg 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdh 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 avg-cpu: %user %nice %system %iowait %steal %idle 0.00 0.00 0.00 0.00 0.00 100.00 Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util sda1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdf 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdg 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdh 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 avg-cpu: %user %nice %system %iowait %steal %idle 0.47 0.00 0.00 0.00 0.00 99.53 Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util sda1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdf 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdg 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdh 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 avg-cpu: %user %nice %system %iowait %steal %idle 3.74 0.00 4.01 0.27 0.00 91.98 Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await svctm %util sda1 0.00 6.00 0.00 5.00 0.00 88.00 17.60 0.01 1.40 1.00 0.50 sdf 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdg 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 sdh 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 [root@mdb1 mongo]# dmesg Linux version 2.6.21.7-2.fc8xen (mockbuild@xenbuilder4.fedora.phx.redhat.com) (gcc version 4.1.2 20070925 (Red Hat 4.1.2-33)) #1 SMP Fri Feb 15 12:34:28 EST 2008 Command line: root=/dev/sda1 ro 4 BIOS-provided physical RAM map: Xen: 0000000000000000 - 00000003c0800000 (usable) Entering add_active_range(0, 0, 3934208) 0 entries of 256 used end_pfn_map = 3934208 Entering add_active_range(0, 0, 3934208) 0 entries of 256 used Zone PFN ranges: DMA 0 -> 3934208 DMA32 3934208 -> 3934208 Normal 3934208 -> 3934208 early_node_map[1] active PFN ranges 0: 0 -> 3934208 On node 0 totalpages: 3934208 DMA zone: 53788 pages used for memmap DMA zone: 0 pages reserved DMA zone: 3880420 pages, LIFO batch:31 DMA32 zone: 0 pages used for memmap Normal zone: 0 pages used for memmap No mptable found. PERCPU: Allocating 26496 bytes of per cpu data Built 1 zonelists. Total pages: 3880420 Kernel command line: root=/dev/sda1 ro 4 Initializing CPU#0 PID hash table entries: 4096 (order: 12, 32768 bytes) Xen reported: 2000.070 MHz processor. Console: colour dummy device 80x25 Dentry cache hash table entries: 2097152 (order: 12, 16777216 bytes) Inode-cache hash table entries: 1048576 (order: 11, 8388608 bytes) Calgary: detecting Calgary via BIOS EBDA area Calgary: Unable to locate Rio Grande table in EBDA - bailing! Software IO TLB disabled Memory: 15413628k/15736832k available (2265k kernel code, 314320k reserved, 1351k data, 200k init) Calibrating delay using timer specific routine.. 4003.39 BogoMIPS (lpj=2001695) Security Framework v1.0.0 initialized SELinux: Initializing. SELinux: Starting in permissive mode selinux_register_security: Registering secondary module capability Capability LSM initialized as secondary Mount-cache hash table entries: 256 CPU: L1 I cache: 32K, L1 D cache: 32K CPU: L2 cache: 256K CPU: L3 cache: 12288K CPU: Physical Processor ID: 0 CPU: Processor Core ID: 1 SMP alternatives: switching to UP code Brought up 1 CPUs sizeof(vma)=168 bytes sizeof(page)=56 bytes sizeof(inode)=560 bytes sizeof(dentry)=208 bytes sizeof(ext3inode)=760 bytes sizeof(buffer_head)=104 bytes sizeof(skbuff)=248 bytes sizeof(task_struct)=1888 bytes NET: Registered protocol family 16 SMP alternatives: switching to SMP code Initializing CPU#1 migration_cost=39 migration_cost=39 Initializing CPU#2 migration_cost=39 Initializing CPU#3 Brought up 4 CPUs PCI: Fatal: No config space access function found PCI: setting up Xen PCI frontend stub ACPI: Interpreter disabled. Linux Plug and Play Support v0.97 (c) Adam Belay pnp: PnP ACPI: disabled xen_mem: Initialising balloon driver. usbcore: registered new interface driver usbfs usbcore: registered new interface driver hub usbcore: registered new device driver usb PCI: System does not support PCI PCI: System does not support PCI NetLabel: Initializing NetLabel: domain hash size = 128 NetLabel: protocols = UNLABELED CIPSOv4 NetLabel: unlabeled traffic allowed by default NET: Registered protocol family 2 IP route cache hash table entries: 524288 (order: 10, 4194304 bytes) TCP established hash table entries: 131072 (order: 9, 3145728 bytes) TCP bind hash table entries: 65536 (order: 8, 1048576 bytes) TCP: Hash tables configured (established 131072 bind 65536) TCP reno registered checking if image is initramfs... it is Freeing initrd memory: 7520k freed audit: initializing netlink socket (disabled) audit(1318559513.094:1): initialized VFS: Disk quotas dquot_6.5.1 Dquot-cache hash table entries: 512 (order 0, 4096 bytes) SELinux: Registering netfilter hooks ksign: Installing public key data Loading keyring io scheduler noop registered io scheduler anticipatory registered io scheduler deadline registered io scheduler cfq registered (default) pci_hotplug: PCI Hot Plug PCI Core version: 0.5 rtc: IRQ 8 is not free. Non-volatile memory driver v1.2 Linux agpgart interface v0.102 (c) Dave Jones RAMDISK driver initialized: 16 RAM disks of 16384K size 4096 blocksize input: Macintosh mouse button emulation as /class/input/input0 Xen virtual console successfully installed as xvc0 Event-channel device installed. usbcore: registered new interface driver hiddev usbcore: registered new interface driver usbhid drivers/usb/input/hid-core.c: v2.6:USB HID core driver PNP: No PS/2 controller found. Probing ports directly. i8042.c: No controller found. mice: PS/2 mouse device common for all mice TCP bic registered Initializing XFRM netlink socket NET: Registered protocol family 1 NET: Registered protocol family 17 XENBUS: Device with no driver: device/vbd/2049 XENBUS: Device with no driver: device/vbd/2128 XENBUS: Device with no driver: device/vif/0 XENBUS: Device with no driver: device/console/0 drivers/rtc/hctosys.c: unable to open rtc device (rtc0) Freeing unused kernel memory: 200k freed Write protecting the kernel read-only data: 981k netfront: Initialising virtual ethernet driver. netfront: device eth0 has copying receive path. xen-vbd: registered block device major 8 sdf: unknown partition table ohci_hcd: 2006 August 04 USB 1.1 'Open' Host Controller (OHCI) Driver USB Universal Host Controller Interface driver v3.0 kjournald starting. Commit interval 5 seconds EXT3-fs: mounted filesystem with ordered data mode. input: PC Speaker as /class/input/input1 Floppy drive(s): fd0 is unknown type 15 (usb?), fd1 is unknown type 15 (usb?) Failed to obtain physical IRQ 6 floppy0: no floppy controllers found lp: driver loaded but no devices found md: Autodetecting RAID arrays. md: autorun ... md: ... autorun DONE. device-mapper: ioctl: 4.11.0-ioctl (2006-10-12) initialised: dm-devel@redhat.com device-mapper: multipath: version 1.0.5 loaded EXT3 FS on sda1, internal journal SGI XFS with ACLs, security attributes, large block/inode numbers, no debug enabled SGI XFS Quota Management subsystem Filesystem "sdf": Disabling barriers, not supported by the underlying device XFS mounting filesystem sdf Ending clean XFS mount for filesystem: sdf IA-32 Microcode Update Driver: v1.14a-xen SCSI subsystem initialized Loading iSCSI transport class v2.0-724. iscsi: registered transport (tcp) iscsi: registered transport (iser) NET: Registered protocol family 10 lo: Disabled Privacy Extensions Mobile IPv6 eth0: no IPv6 routers present sdg: unknown partition table Filesystem "sdg": Disabling barriers, not supported by the underlying device XFS mounting filesystem sdg Ending clean XFS mount for filesystem: sdg sdh: unknown partition table Filesystem "sdh": Disabling barriers, not supported by the underlying device XFS mounting filesystem sdh Ending clean XFS mount for filesystem: sdh Filesystem "sdh": Disabling barriers, not supported by the underlying device XFS mounting filesystem sdh Ending clean XFS mount for filesystem: sdh [root@mdb1 mongo]# df -k Filesystem 1K-blocks Used Available Use% Mounted on /dev/sda1 132111844 2194592 124550804 2% / none 7868180 0 7868180 0% /dev/shm /dev/sdf 134152192 32471844 101680348 25% /data /dev/sdh 536739840 127222372 409517468 24% /backups [root@mdb1 mongo]# cd /var/log/ [root@mdb1 log]# ls -lte ls: invalid option -- e Try `ls --help' for more information. [root@mdb1 log]# ls -ltr total 428 drwxr-xr-x 2 root root 4096 Nov 11 2007 conman.old drwxr-xr-x 2 root root 4096 Nov 11 2007 conman drwx------ 2 root root 4096 Jan 20 2009 ppp drwxr-xr-x 2 root root 4096 Sep 15 2010 pm drwxr-x--- 2 root root 4096 Mar 5 2011 audit drwxr-xr-x 2 lp sys 4096 Mar 30 2011 cups drwxr-x--- 2 puppet puppet 4096 May 5 15:52 puppet -rw------- 1 root root 0 Aug 30 11:38 tallylog -rw------- 1 root root 0 Aug 30 11:38 spooler -rw-r--r-- 1 root root 0 Aug 30 11:38 mcollective.log -rw------- 1 root utmp 0 Aug 30 11:38 btmp -rw------- 1 root root 0 Aug 30 11:38 boot.log -rw------- 1 root root 224032 Oct 13 22:28 faillog -rw-r--r-- 1 root root 5979 Oct 13 22:32 dmesg -rw-r--r-- 1 root root 1906 Oct 13 22:32 brcm-iscsi.log -rw-r--r-- 1 root root 4451 Oct 16 04:02 rpmpkgs-20111016.gz -rw-r--r-- 1 root root 5547 Oct 20 04:02 secure-20111020.gz -rw-r--r-- 1 root root 1354 Oct 20 04:02 messages-20111020.gz -rw-r--r-- 1 root root 2123 Oct 20 04:02 maillog-20111020.gz -rw-r--r-- 1 root root 13837 Oct 20 04:02 cron-20111020.gz -rw-r--r-- 1 root root 5648 Oct 21 04:02 secure-20111021.gz -rw-r--r-- 1 root root 1538 Oct 21 04:02 messages-20111021.gz -rw-r--r-- 1 root root 2097 Oct 21 04:02 maillog-20111021.gz -rw-r--r-- 1 root root 13836 Oct 21 04:02 cron-20111021.gz -rw-r--r-- 1 root root 13447 Oct 21 13:22 kern -rw-r--r-- 1 root root 4994 Oct 22 04:02 secure-20111022.gz -rw-r--r-- 1 root root 1589 Oct 22 04:02 messages-20111022.gz -rw-r--r-- 1 root root 2103 Oct 22 04:02 maillog-20111022.gz -rw-r--r-- 1 root root 13898 Oct 22 04:02 cron-20111022.gz -rw-r--r-- 1 root root 5189 Oct 23 04:02 secure-20111023.gz -rw-r--r-- 1 root root 4436 Oct 23 04:02 rpmpkgs-20111023.gz -rw-r--r-- 1 root root 1555 Oct 23 04:02 messages-20111023.gz -rw-r--r-- 1 root root 2092 Oct 23 04:02 maillog-20111023.gz -rw-r--r-- 1 root root 13883 Oct 23 04:02 cron-20111023.gz -rw-r--r-- 1 root root 4965 Oct 24 04:02 secure-20111024.gz -rw-r--r-- 1 root root 1487 Oct 24 04:02 messages-20111024.gz -rw-r--r-- 1 root root 2118 Oct 24 04:02 maillog-20111024.gz -rw-r--r-- 1 root root 13968 Oct 24 04:02 cron-20111024.gz -rw-r--r-- 1 root root 4965 Oct 25 04:02 secure-20111025.gz -rw-r--r-- 1 root root 1413 Oct 25 04:02 messages-20111025.gz -rw-r--r-- 1 root root 2134 Oct 25 04:02 maillog-20111025.gz -rw-r--r-- 1 root root 13806 Oct 25 04:02 cron-20111025.gz -rw-r--r-- 1 root root 1746 Oct 25 14:10 yum.log drwxr-xr-x 2 root root 4096 Oct 26 00:00 sa -rw-r--r-- 1 root root 5263 Oct 26 04:02 secure-20111026.gz drwxr-xr-x 2 mongod mongod 4096 Oct 26 04:02 mongo -rw-r--r-- 1 root root 1490 Oct 26 04:02 messages-20111026.gz -rw-r--r-- 1 root root 2141 Oct 26 04:02 maillog-20111026.gz -rw-r--r-- 1 root root 13980 Oct 26 04:02 cron-20111026.gz drwxr-xr-x 2 root root 4096 Oct 26 04:02 chef -rw-r--r-- 1 root root 15803 Oct 26 04:02 rpmpkgs -rw-r--r-- 1 root root 589 Oct 26 04:02 maillog -rw-rw-r-- 1 root utmp 28416 Oct 26 04:32 wtmp -rw-r--r-- 1 root root 2924088 Oct 26 04:32 lastlog -rw-r--r-- 1 root root 665 Oct 26 04:33 messages -rw-r--r-- 1 root root 1456 Oct 26 04:34 secure -rw-r--r-- 1 root root 4938 Oct 26 04:34 cron [root@mdb1 log]# tail messages Oct 26 04:02:06 mdb1.prod.prv syslog-ng[1211]: Configuration reload request received, reloading configuration; Oct 26 04:02:06 mdb1.prod.prv syslog-ng[1211]: Syslog connection established; fd='11', server='AF_INET(10.240.46.127:514)', local='AF_INET(0.0.0.0:0)' Oct 26 04:07:33 mdb1.prod.prv ntpd[1258]: frequency error -512 PPM exceeds tolerance 500 PPM Oct 26 04:16:08 mdb1.prod.prv ntpd[1258]: frequency error -512 PPM exceeds tolerance 500 PPM Oct 26 04:22:06 mdb1.prod.prv -- MARK -- Oct 26 04:33:15 mdb1.prod.prv ntpd[1258]: synchronized to 10.126.38.185, stratum 2 Oct 26 04:33:15 mdb1.prod.prv ntpd[1258]: frequency error -512 PPM exceeds tolerance 500 PPM [root@mdb1 log]# date Wed Oct 26 04:34:52 EDT 2011 [root@mdb1 log]# tail cron Oct 26 04:27:01 mdb1.prod.prv crond[7784]: (root) CMD (for FILE in `ls /opt/sm/bin/gmetric-*`; do command $FILE; done >/dev/null 2>&1) Oct 26 04:28:01 mdb1.prod.prv crond[7850]: (root) CMD (for FILE in `ls /opt/sm/bin/gmetric-*`; do command $FILE; done >/dev/null 2>&1) Oct 26 04:29:02 mdb1.prod.prv crond[7906]: (root) CMD (for FILE in `ls /opt/sm/bin/gmetric-*`; do command $FILE; done >/dev/null 2>&1) Oct 26 04:30:01 mdb1.prod.prv crond[7982]: (root) CMD (/usr/lib64/sa/sa1 1 1) top - 04:37:26 up 12 days, 5:56, 1 user, load average: 0.05, 0.01, 0.04 Tasks: 275 total, 1 running, 273 sleeping, 0 stopped, 1 zombie Cpu(s): 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st Mem: 15736360k total, 15674760k used, 61600k free, 22904k buffers Swap: 0k total, 0k used, 0k free, 13951448k cached PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 2809 mongod 16 0 54.9g 8.3g 7.5g S 0.0 55.2 616:05.50 mongod 5019 root 24 0 150m 58m 3668 S 0.0 0.4 0:06.32 chef-client 4812 root 16 0 44032 11m 3408 S 0.0 0.1 0:16.51 mongodump 1258 ntp 15 0 35120 8744 7324 S 0.0 0.1 0:00.13 ntpd 8178 root 15 0 105m 4684 3604 S 0.0 0.0 0:00.04 sshd 952 root 5 -10 12624 4440 3152 S 0.0 0.0 0:00.00 iscsid 5610 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.03 mongo 5935 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.04 mongo 6146 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.03 mongo 6593 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.03 mongo 6978 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.04 mongo 7129 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.04 mongo 8106 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.04 mongo 8266 root 25 0 28188 4348 3140 S 0.0 0.0 0:00.04 mongo 8425 root 25 0 28184 4348 3140 S 0.0 0.0 0:00.03 mongo 5064 root 24 0 28184 4344 3140 S 0.0 0.0 0:00.03 mongo 5668 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.04 mongo 5759 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.03 mongo 5817 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.04 mongo 5991 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.03 mongo 6086 root 25 0 28184 4344 3140 S 0.0 0.0 0:00.03 mongo 6649 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.03 mongo 6800 root 25 0 28184 4344 3140 S 0.0 0.0 0:00.04 mongo 6864 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.03 mongo 7781 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.03 mongo 7845 root 25 0 28184 4344 3140 S 0.0 0.0 0:00.04 mongo 7903 root 25 0 28188 4344 3140 S 0.0 0.0 0:00.04 mongo 7959 root 25 0 28184 4344 3140 S 0.0 0.0 0:00.04 mongo 8172 root 25 0 28184 4344 3140 S 0.0 0.0 0:00.03 mongo 8328 root 25 0 28184 4344 3140 S 0.0 0.0 0:00.03 mongo 8883 root 25 0 28184 4344 3140 S 0.0 0.0 0:00.03 mongo 5881 root 25 0 28184 4340 3140 S 0.0 0.0 0:00.04 mongo [root@mdb1 log]# cd /backups [root@mdb1 backups]# ls daily hourly manual monthly weekly [root@mdb1 backups]# cd daily/ [root@mdb1 daily]# ls -ltr total 28465968 -rw-r--r-- 2 root root 1452101010 Oct 9 12:14 mongo-100911-12.tgz -rw-r--r-- 1 root root 1581663644 Oct 12 12:19 mongo-101211-12.tgz -rw-r--r-- 1 root root 1620074029 Oct 13 12:19 mongo-101311-12.tgz -rw-r--r-- 1 root root 1652742862 Oct 14 12:11 mongo-101411-12.tgz -rw-r--r-- 1 root root 1691394530 Oct 15 12:11 mongo-101511-12.tgz -rw-r--r-- 1 root root 1728778255 Oct 16 12:12 mongo-101611-12.tgz -rw-r--r-- 1 root root 1767071403 Oct 17 12:13 mongo-101711-12.tgz -rw-r--r-- 1 root root 1831516727 Oct 18 12:13 mongo-101811-12.tgz -rw-r--r-- 1 root root 1874230122 Oct 19 12:14 mongo-101911-12.tgz -rw-r--r-- 1 root root 1895498625 Oct 20 12:14 mongo-102011-12.tgz -rw-r--r-- 1 root root 1938374169 Oct 22 12:13 mongo-102211-12.tgz -rw-r--r-- 1 root root 1961713104 Oct 23 12:14 mongo-102311-12.tgz -rw-r--r-- 1 root root 1992019655 Oct 24 12:15 mongo-102411-12.tgz -rw-r--r-- 1 root root 2047232892 Oct 25 12:16 mongo-102511-12.tgz -rw-r--r-- 1 root root 2052124179 Oct 25 14:28 mongo-102511-14.tgz -rw-r--r-- 1 root root 2062576222 Oct 26 00:16 mongo-102611-00.tgz [root@mdb1 daily]# ps -aef|grep mongo mongod 2809 1 3 Oct13 ? 10:16:05 /usr/bin/mongod -f /etc/mongod.conf --rest --journal --replSet=mercury root 4812 4784 0 04:00 ? 00:00:16 mongodump -o /backups/hourly/102611-04 root 5061 4971 0 04:02 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5063 5061 0 04:02 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5064 5063 0 04:02 ? 00:00:00 /usr/bin/mongo root 5608 5557 0 04:03 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5609 5608 0 04:03 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5610 5609 0 04:03 ? 00:00:00 /usr/bin/mongo root 5666 5615 0 04:04 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5667 5666 0 04:04 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5668 5667 0 04:04 ? 00:00:00 /usr/bin/mongo root 5757 5688 0 04:05 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5758 5757 0 04:05 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5759 5758 0 04:05 ? 00:00:00 /usr/bin/mongo root 5815 5764 0 04:06 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5816 5815 0 04:06 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5817 5816 0 04:06 ? 00:00:00 /usr/bin/mongo root 5879 5820 0 04:07 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5880 5879 0 04:07 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5881 5880 0 04:07 ? 00:00:00 /usr/bin/mongo root 5933 5886 0 04:08 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5934 5933 0 04:08 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5935 5934 0 04:08 ? 00:00:00 /usr/bin/mongo root 5989 5942 0 04:09 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5990 5989 0 04:09 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5991 5990 0 04:09 ? 00:00:00 /usr/bin/mongo root 6084 6019 0 04:10 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6085 6084 0 04:10 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6086 6085 0 04:10 ? 00:00:00 /usr/bin/mongo root 6144 6093 0 04:11 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6145 6144 0 04:11 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6146 6145 0 04:11 ? 00:00:00 /usr/bin/mongo root 6206 6151 0 04:12 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6207 6206 0 04:12 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6208 6207 0 04:12 ? 00:00:00 /usr/bin/mongo root 6591 6540 0 04:13 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6592 6591 0 04:13 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6593 6592 0 04:13 ? 00:00:00 /usr/bin/mongo root 6647 6598 0 04:14 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6648 6647 0 04:14 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6649 6648 0 04:14 ? 00:00:00 /usr/bin/mongo root 6740 6671 0 04:15 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6741 6740 0 04:15 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6742 6741 0 04:15 ? 00:00:00 /usr/bin/mongo root 6798 6747 0 04:16 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6799 6798 0 04:16 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6800 6799 0 04:16 ? 00:00:00 /usr/bin/mongo root 6862 6803 0 04:17 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6863 6862 0 04:17 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6864 6863 0 04:17 ? 00:00:00 /usr/bin/mongo root 6920 6869 0 04:18 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6921 6920 0 04:18 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6922 6921 0 04:18 ? 00:00:00 /usr/bin/mongo root 6976 6925 0 04:19 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6977 6976 0 04:19 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6978 6977 0 04:19 ? 00:00:00 /usr/bin/mongo root 7071 7002 0 04:20 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7072 7071 0 04:20 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7073 7072 0 04:20 ? 00:00:00 /usr/bin/mongo root 7127 7076 0 04:21 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7128 7127 0 04:21 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7129 7128 0 04:21 ? 00:00:00 /usr/bin/mongo root 7193 7134 0 04:22 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7194 7193 0 04:22 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7195 7194 0 04:22 ? 00:00:00 /usr/bin/mongo root 7249 7198 0 04:23 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7250 7249 0 04:23 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7251 7250 0 04:23 ? 00:00:00 /usr/bin/mongo root 7307 7256 0 04:24 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7308 7307 0 04:24 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7309 7308 0 04:24 ? 00:00:00 /usr/bin/mongo root 7596 7466 0 04:25 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7597 7596 0 04:25 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7598 7597 0 04:25 ? 00:00:00 /usr/bin/mongo root 7779 7728 0 04:26 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7780 7779 0 04:26 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7781 7780 0 04:26 ? 00:00:00 /usr/bin/mongo root 7843 7784 0 04:27 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7844 7843 0 04:27 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7845 7844 0 04:27 ? 00:00:00 /usr/bin/mongo root 7901 7850 0 04:28 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7902 7901 0 04:28 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7903 7902 0 04:28 ? 00:00:00 /usr/bin/mongo root 7957 7906 0 04:29 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7958 7957 0 04:29 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7959 7958 0 04:29 ? 00:00:00 /usr/bin/mongo root 8048 7983 0 04:30 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8049 8048 0 04:30 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8050 8049 0 04:30 ? 00:00:00 /usr/bin/mongo root 8104 8057 0 04:31 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8105 8104 0 04:31 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8106 8105 0 04:31 ? 00:00:00 /usr/bin/mongo root 8170 8115 0 04:32 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8171 8170 0 04:32 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8172 8171 0 04:32 ? 00:00:00 /usr/bin/mongo root 8264 8213 0 04:33 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8265 8264 0 04:33 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8266 8265 0 04:33 ? 00:00:00 /usr/bin/mongo root 8326 8275 0 04:34 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8327 8326 0 04:34 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8328 8327 0 04:34 ? 00:00:00 /usr/bin/mongo root 8423 8354 0 04:35 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8424 8423 0 04:35 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8425 8424 0 04:35 ? 00:00:00 /usr/bin/mongo root 8489 8437 0 04:36 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8490 8489 0 04:36 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8491 8490 0 04:36 ? 00:00:00 /usr/bin/mongo root 8881 8822 0 04:37 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8882 8881 0 04:37 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8883 8882 0 04:37 ? 00:00:00 /usr/bin/mongo root 8888 8180 0 04:37 pts/0 00:00:00 grep mongo [root@mdb1 daily]# cd /backups/hourly/ [root@mdb1 hourly]# ls -ltr total 84876860 -rw-r--r-- 1 root root 1982198049 Oct 24 04:13 mongo-102411-04.tgz -rw-r--r-- 1 root root 1982304842 Oct 24 05:13 mongo-102411-05.tgz -rw-r--r-- 1 root root 1984835941 Oct 24 06:16 mongo-102411-06.tgz -rw-r--r-- 1 root root 1989199554 Oct 24 07:14 mongo-102411-07.tgz -rw-r--r-- 1 root root 1989341286 Oct 24 08:14 mongo-102411-08.tgz -rw-r--r-- 1 root root 1989514430 Oct 24 09:14 mongo-102411-09.tgz -rw-r--r-- 1 root root 1989644238 Oct 24 10:14 mongo-102411-10.tgz -rw-r--r-- 1 root root 1989752403 Oct 24 11:14 mongo-102411-11.tgz -rw-r--r-- 1 root root 1996852446 Oct 24 13:13 mongo-102411-13.tgz -rw-r--r-- 1 root root 1996981598 Oct 24 14:15 mongo-102411-14.tgz -rw-r--r-- 1 root root 1997106861 Oct 24 15:14 mongo-102411-15.tgz -rw-r--r-- 1 root root 1997267330 Oct 24 16:14 mongo-102411-16.tgz -rw-r--r-- 1 root root 1997447791 Oct 24 17:14 mongo-102411-17.tgz -rw-r--r-- 1 root root 1999814454 Oct 24 18:17 mongo-102411-18.tgz -rw-r--r-- 1 root root 2005359999 Oct 24 19:15 mongo-102411-19.tgz -rw-r--r-- 1 root root 2005515061 Oct 24 20:16 mongo-102411-20.tgz -rw-r--r-- 1 root root 2005649261 Oct 24 21:16 mongo-102411-21.tgz -rw-r--r-- 1 root root 2005781373 Oct 24 22:15 mongo-102411-22.tgz -rw-r--r-- 1 root root 2005906668 Oct 24 23:15 mongo-102411-23.tgz -rw-r--r-- 1 root root 2008196253 Oct 25 00:15 mongo-102511-00.tgz -rw-r--r-- 1 root root 2012730394 Oct 25 01:14 mongo-102511-01.tgz -rw-r--r-- 1 root root 2012867347 Oct 25 02:14 mongo-102511-02.tgz -rw-r--r-- 1 root root 2013000194 Oct 25 03:15 mongo-102511-03.tgz -rw-r--r-- 1 root root 2013093853 Oct 25 04:14 mongo-102511-04.tgz -rw-r--r-- 1 root root 2013219464 Oct 25 05:18 mongo-102511-05.tgz -rw-r--r-- 1 root root 2015850831 Oct 25 06:17 mongo-102511-06.tgz -rw-r--r-- 1 root root 2020462064 Oct 25 07:18 mongo-102511-07.tgz -rw-r--r-- 1 root root 2044572997 Oct 25 08:17 mongo-102511-08.tgz -rw-r--r-- 1 root root 2044699660 Oct 25 09:15 mongo-102511-09.tgz -rw-r--r-- 1 root root 2044818470 Oct 25 10:15 mongo-102511-10.tgz -rw-r--r-- 1 root root 2044961151 Oct 25 11:15 mongo-102511-11.tgz -rw-r--r-- 1 root root 2052264939 Oct 25 15:15 mongo-102511-15.tgz -rw-r--r-- 1 root root 2052428498 Oct 25 16:15 mongo-102511-16.tgz -rw-r--r-- 1 root root 2052592807 Oct 25 17:15 mongo-102511-17.tgz -rw-r--r-- 1 root root 2054912595 Oct 25 18:16 mongo-102511-18.tgz -rw-r--r-- 1 root root 2059767562 Oct 25 19:14 mongo-102511-19.tgz -rw-r--r-- 1 root root 2059911081 Oct 25 20:15 mongo-102511-20.tgz -rw-r--r-- 1 root root 2060007399 Oct 25 21:14 mongo-102511-21.tgz -rw-r--r-- 1 root root 2060124999 Oct 25 22:15 mongo-102511-22.tgz -rw-r--r-- 1 root root 2060247704 Oct 25 23:15 mongo-102511-23.tgz -rw-r--r-- 1 root root 2066971416 Oct 26 01:15 mongo-102611-01.tgz -rw-r--r-- 1 root root 2067528915 Oct 26 02:15 mongo-102611-02.tgz -rw-r--r-- 1 root root 2068053589 Oct 26 03:16 mongo-102611-03.tgz drwxr-xr-x 4 root root 49 Oct 26 04:00 102611-04 [root@mdb1 hourly]# cd 102611-04/ [root@mdb1 102611-04]# ls -la total 8 drwxr-xr-x 4 root root 49 Oct 26 04:00 . drwxr-xr-x 3 root root 4096 Oct 26 04:00 .. drwxr-xr-x 2 root root 4096 Oct 26 04:00 assets_prod drwxr-xr-x 2 root root 145 Oct 26 04:01 notifications_prod [root@mdb1 102611-04]# cd notifications_prod/ [root@mdb1 notifications_prod]# ls -ltr total 7028768 -rw-r--r-- 1 root root 3785875981 Oct 26 04:01 ActivityEvent.bson -rw-r--r-- 1 root root 1725 Oct 26 04:01 system.indexes.bson -rw-r--r-- 1 root root 705 Oct 26 04:01 Sequences.bson -rw-r--r-- 1 root root 6117766 Oct 26 04:01 Notification.bson -rw-r--r-- 1 root root 1300 Oct 26 04:01 mongeez.bson -rw-r--r-- 1 root root 3405402415 Oct 26 04:02 EmailArchive.bson [root@mdb1 notifications_prod]# ls -ltr total 7028768 -rw-r--r-- 1 root root 3785875981 Oct 26 04:01 ActivityEvent.bson -rw-r--r-- 1 root root 1725 Oct 26 04:01 system.indexes.bson -rw-r--r-- 1 root root 705 Oct 26 04:01 Sequences.bson -rw-r--r-- 1 root root 6117766 Oct 26 04:01 Notification.bson -rw-r--r-- 1 root root 1300 Oct 26 04:01 mongeez.bson -rw-r--r-- 1 root root 3405402415 Oct 26 04:02 EmailArchive.bson [root@mdb1 notifications_prod]# ls -ltr total 7028768 -rw-r--r-- 1 root root 3785875981 Oct 26 04:01 ActivityEvent.bson -rw-r--r-- 1 root root 1725 Oct 26 04:01 system.indexes.bson -rw-r--r-- 1 root root 705 Oct 26 04:01 Sequences.bson -rw-r--r-- 1 root root 6117766 Oct 26 04:01 Notification.bson -rw-r--r-- 1 root root 1300 Oct 26 04:01 mongeez.bson -rw-r--r-- 1 root root 3405402415 Oct 26 04:02 EmailArchive.bson [root@mdb1 notifications_prod]# killall mongodump [root@mdb1 notifications_prod]# killall mongodump mongodump: no process killed [root@mdb1 notifications_prod]# ls -l total 7028728 -rw-r--r-- 1 root root 3785875981 Oct 26 04:01 ActivityEvent.bson -rw-r--r-- 1 root root 3405402415 Oct 26 04:02 EmailArchive.bson -rw-r--r-- 1 root root 1300 Oct 26 04:01 mongeez.bson -rw-r--r-- 1 root root 6117766 Oct 26 04:01 Notification.bson -rw-r--r-- 1 root root 705 Oct 26 04:01 Sequences.bson -rw-r--r-- 1 root root 1725 Oct 26 04:01 system.indexes.bson [root@mdb1 notifications_prod]# df -k Filesystem 1K-blocks Used Available Use% Mounted on /dev/sda1 132111844 2194640 124550756 2% / none 7868180 0 7868180 0% /dev/shm /dev/sdf 134152192 32471844 101680348 25% /data /dev/sdh 536739840 127301396 409438444 24% /backups [root@mdb1 notifications_prod]# ps -aef|grep mongo mongod 2809 1 3 Oct13 ? 10:16:05 /usr/bin/mongod -f /etc/mongod.conf --rest --journal --replSet=mercury root 5061 4971 0 04:02 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5063 5061 0 04:02 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5064 5063 0 04:02 ? 00:00:00 /usr/bin/mongo root 5608 5557 0 04:03 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5609 5608 0 04:03 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5610 5609 0 04:03 ? 00:00:00 /usr/bin/mongo root 5666 5615 0 04:04 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5667 5666 0 04:04 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5668 5667 0 04:04 ? 00:00:00 /usr/bin/mongo root 5757 5688 0 04:05 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5758 5757 0 04:05 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5759 5758 0 04:05 ? 00:00:00 /usr/bin/mongo root 5815 5764 0 04:06 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5816 5815 0 04:06 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5817 5816 0 04:06 ? 00:00:00 /usr/bin/mongo root 5879 5820 0 04:07 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5880 5879 0 04:07 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5881 5880 0 04:07 ? 00:00:00 /usr/bin/mongo root 5933 5886 0 04:08 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5934 5933 0 04:08 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5935 5934 0 04:08 ? 00:00:00 /usr/bin/mongo root 5989 5942 0 04:09 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5990 5989 0 04:09 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 5991 5990 0 04:09 ? 00:00:00 /usr/bin/mongo root 6084 6019 0 04:10 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6085 6084 0 04:10 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6086 6085 0 04:10 ? 00:00:00 /usr/bin/mongo root 6144 6093 0 04:11 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6145 6144 0 04:11 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6146 6145 0 04:11 ? 00:00:00 /usr/bin/mongo root 6206 6151 0 04:12 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6207 6206 0 04:12 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6208 6207 0 04:12 ? 00:00:00 /usr/bin/mongo root 6591 6540 0 04:13 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6592 6591 0 04:13 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6593 6592 0 04:13 ? 00:00:00 /usr/bin/mongo root 6647 6598 0 04:14 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6648 6647 0 04:14 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6649 6648 0 04:14 ? 00:00:00 /usr/bin/mongo root 6740 6671 0 04:15 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6741 6740 0 04:15 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6742 6741 0 04:15 ? 00:00:00 /usr/bin/mongo root 6798 6747 0 04:16 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6799 6798 0 04:16 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6800 6799 0 04:16 ? 00:00:00 /usr/bin/mongo root 6862 6803 0 04:17 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6863 6862 0 04:17 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6864 6863 0 04:17 ? 00:00:00 /usr/bin/mongo root 6920 6869 0 04:18 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6921 6920 0 04:18 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6922 6921 0 04:18 ? 00:00:00 /usr/bin/mongo root 6976 6925 0 04:19 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6977 6976 0 04:19 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 6978 6977 0 04:19 ? 00:00:00 /usr/bin/mongo root 7071 7002 0 04:20 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7072 7071 0 04:20 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7073 7072 0 04:20 ? 00:00:00 /usr/bin/mongo root 7127 7076 0 04:21 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7128 7127 0 04:21 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7129 7128 0 04:21 ? 00:00:00 /usr/bin/mongo root 7193 7134 0 04:22 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7194 7193 0 04:22 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7195 7194 0 04:22 ? 00:00:00 /usr/bin/mongo root 7249 7198 0 04:23 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7250 7249 0 04:23 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7251 7250 0 04:23 ? 00:00:00 /usr/bin/mongo root 7307 7256 0 04:24 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7308 7307 0 04:24 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7309 7308 0 04:24 ? 00:00:00 /usr/bin/mongo root 7596 7466 0 04:25 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7597 7596 0 04:25 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7598 7597 0 04:25 ? 00:00:00 /usr/bin/mongo root 7779 7728 0 04:26 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7780 7779 0 04:26 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7781 7780 0 04:26 ? 00:00:00 /usr/bin/mongo root 7843 7784 0 04:27 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7844 7843 0 04:27 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7845 7844 0 04:27 ? 00:00:00 /usr/bin/mongo root 7901 7850 0 04:28 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7902 7901 0 04:28 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7903 7902 0 04:28 ? 00:00:00 /usr/bin/mongo root 7957 7906 0 04:29 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7958 7957 0 04:29 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 7959 7958 0 04:29 ? 00:00:00 /usr/bin/mongo root 8048 7983 0 04:30 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8049 8048 0 04:30 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8050 8049 0 04:30 ? 00:00:00 /usr/bin/mongo root 8104 8057 0 04:31 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8105 8104 0 04:31 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8106 8105 0 04:31 ? 00:00:00 /usr/bin/mongo root 8170 8115 0 04:32 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8171 8170 0 04:32 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8172 8171 0 04:32 ? 00:00:00 /usr/bin/mongo root 8264 8213 0 04:33 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8265 8264 0 04:33 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8266 8265 0 04:33 ? 00:00:00 /usr/bin/mongo root 8326 8275 0 04:34 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8327 8326 0 04:34 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8328 8327 0 04:34 ? 00:00:00 /usr/bin/mongo root 8423 8354 0 04:35 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8424 8423 0 04:35 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8425 8424 0 04:35 ? 00:00:00 /usr/bin/mongo root 8489 8437 0 04:36 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8490 8489 0 04:36 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8491 8490 0 04:36 ? 00:00:00 /usr/bin/mongo root 8881 8822 0 04:37 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8882 8881 0 04:37 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8883 8882 0 04:37 ? 00:00:00 /usr/bin/mongo root 8943 8892 0 04:38 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8944 8943 0 04:38 ? 00:00:00 /bin/bash /opt/sm/bin/gmetric-mongodb.sh root 8945 8944 0 04:38 ? 00:00:00 /usr/bin/mongo root 8953 4784 0 04:38 ? 00:00:00 tar cfz /backups/hourly/mongo-102611-04.tgz /backups/hourly/102611-04 root 8970 8180 0 04:39 pts/0 00:00:00 grep mongo [root@mdb1 notifications_prod]# mongo MongoDB shell version: 1.8.3 connecting to: test [root@mdb1 notifications_prod]# killall mongo [root@mdb1 notifications_prod]# killall mongo mongo: no process killed [root@mdb1 notifications_prod]# ps -aef|grep mongo mongod 2809 1 3 Oct13 ? 10:16:05 /usr/bin/mongod -f /etc/mongod.conf --rest --journal --replSet=mercury root 8953 4784 0 04:38 ? 00:00:00 tar cfz /backups/hourly/mongo-102611-04.tgz /backups/hourly/102611-04 root 9333 8180 0 04:39 pts/0 00:00:00 grep mongo [root@mdb1 notifications_prod]# df -k Filesystem 1K-blocks Used Available Use% Mounted on /dev/sda1 132111844 2194492 124550904 2% / none 7868180 0 7868180 0% /dev/shm /dev/sdf 134152192 32471844 101680348 25% /data /dev/sdh 536739840 127436212 409303628 24% /backups [root@mdb1 notifications_prod]# mongo MongoDB shell version: 1.8.3 connecting to: test ----- [root@mdb1 ~]# cat /etc/redhat-release CentOS release 5.6 (Final) [root@mdb1 ~]# cat /etc/mongod.conf # mongo.conf #where to log logpath=/var/log/mongo/mongod.log logappend=true # fork and run in background fork = true #port = 27017 dbpath=/data/mongo # Enables periodic logging of CPU utilization and I/O wait #cpu = true # Turn on/off security. Off is currently the default noauth = true #auth = true # Verbose logging output. #verbose = true # Inspect all client data for validity on receipt (useful for # developing drivers) #objcheck = true # Enable db quota management #quota = true # Set oplogging level where n is # 0=off (default) # 1=W # 2=R # 3=both # 7=W+some reads #oplog = 0 # Diagnostic/debugging option #nocursors = true # Ignore query hints #nohints = true # Disable the HTTP interface (Defaults to localhost:27018). #nohttpinterface = true # Turns off server-side scripting. This will result in greatly limited # functionality #noscripting = true # Turns off table scans. Any query that would do a table scan fails. #notablescan = true # Disable data file preallocation. #noprealloc = true # Specify .ns file size for new databases. # nssize = # Accout token for Mongo monitoring server. #mms-token = # Server name for Mongo monitoring server. #mms-name = # Ping interval for Mongo monitoring server. #mms-interval = # Replication Options # in replicated mongo databases, specify here whether this is a slave or master # autoresync = true # Custom size for replication operation log. #oplogSize = # Size limit for in-memory storage of op ids. #opIdMem = [root@mdb1 ~]# cat /etc/sysconfig/mongod #6--- OPTIONS="$OPTIONS --rest --journal --replSet=mercury" [root@mdb1 ~]# cat /etc/fstab /dev/sda1 / ext3 noatime 1 1 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 none /proc proc defaults 0 0 none /sys sysfs defaults 0 0 /dev/sdf /data xfs noatime 0 0 /dev/sdh /backups xfs noatime 0 0 /dev/sda3 swap swap defaults 0 0 [root@mdb1 ~]#