From a7796bec1be01b51f400e9a3a698535b12b636aa Mon Sep 17 00:00:00 2001
From: Marius Burkard <m.burkard@pixcept.de>
Date: Thu, 11 Feb 2016 13:44:27 -0500
Subject: [PATCH] Merge branch 'stable-3.1'
---
server/mods-available/rescue_core_module.inc.php | 233 +++++++++++++++++++++++++++++++++++++++++-----------------
1 files changed, 165 insertions(+), 68 deletions(-)
diff --git a/server/mods-available/rescue_core_module.inc.php b/server/mods-available/rescue_core_module.inc.php
index f1c34ca..5698273 100644
--- a/server/mods-available/rescue_core_module.inc.php
+++ b/server/mods-available/rescue_core_module.inc.php
@@ -35,12 +35,16 @@
/* No actions at this time. maybe later... */
var $actions_available = array();
/**
- * The monitoring-Data of this module.
+ * The monitoring-Data of this module.
* [0] are the actual data, [1] are the data 1 minnute ago [2] are teh data 2 minuntes...
*/
+
+
private $_monitoringData = array();
+
/** The rescue-Data of this module. */
private $_rescueData = array();
+
/**
* This function is called during ispconfig installation to determine
@@ -50,12 +54,14 @@
return true;
}
+
/**
* This function is called when the module is loaded
*/
function onLoad() {
$this->_doRescue();
}
+
/**
* This function is called when a change in one of the registered tables is detected.
@@ -73,8 +79,8 @@
* do nothing, if the rescue-system is not enabled
*/
global $conf;
- if ((!isset($conf['serverconfig']['rescue']['try_rescue'])) ||
- ((isset($conf['serverconfig']['rescue']['try_rescue'])) && ($conf['serverconfig']['rescue']['try_rescue'] !='y'))){
+ if ((!isset($conf['serverconfig']['rescue']['try_rescue'])) ||
+ ((isset($conf['serverconfig']['rescue']['try_rescue'])) && ($conf['serverconfig']['rescue']['try_rescue'] !='y'))){
return;
}
@@ -87,17 +93,22 @@
* Next we get the rescue data needed for rescuing the system
*/
$this->_rescueData = $this->_getRescueData();
-
+
/*
- * rescue mysql if needed (maybe apache depends on mysql, so try this first!)
+ * rescue MongoDB if needed
+ */
+ $this->_rescueMongoDB();
+
+ /*
+ * rescue mysql if needed (maybe httpd depends on mysql, so try this first!)
*/
$this->_rescueMySql();
-
+
/*
- * rescue apache if needed
+ * rescue httpd if needed
*/
- $this->_rescueApache();
-
+ $this->_rescueHttpd();
+
/*
* The last step is to save the rescue-data
*/
@@ -112,12 +123,12 @@
*/
private function _getMonitoringData() {
global $app;
-
+
$dataFilename = dirname(__FILE__) . "/../temp/rescue_module_monitoringdata.ser.txt";
-
+
/*
* If the file containing the data is too old (older than 5 minutes) it is better to
- * delete it, because it could be, that the server was down for some times and so the data
+ * delete it, because it could be, that the server was down for some times and so the data
* are outdated
*/
if (file_exists($dataFilename) && (filemtime($dataFilename) < (time() - 5 * 60))) {
@@ -132,7 +143,7 @@
} else {
$data = array();
}
-
+
/*
* $temp[0] was the data of the last monitoring (means 1 minute ago), $temp[1] is the data
* 2 minutes ago and so on. Now we have make place for the newest data...
@@ -144,32 +155,32 @@
for ($i = $max; $i > 0; $i--){
$data[$i] = $data[$i -1];
}
-
+
/*
* we need the monitoring tools
*/
$app->load('monitor_tools');
$tools = new monitor_tools();
-
+
/*
* Call the needed Monitoring-step and get the data
*/
$tmp[0] = $tools->monitorServices();
-
+
/* Add the data at the FIRST position of the history */
$data[0] = $tmp;
-
+
/*
* We have the newest monitoring data. Save it!
* (and protect it, because there may be sensible data in it)
*/
file_put_contents($dataFilename, serialize($data));
chmod($dataFilename, 0600);
-
+
/* Thats it */
return $data;
}
-
+
/**
* This gets the rescue-Data, needed for rescuing the system.
* Because we can not be 100% sure, that the mysql-DB is up and running, so we use the
@@ -181,7 +192,7 @@
/*
* If the file containing the data is too old (older than 5 minutes) it is better to
- * delete it, because it could be, that the server was down for some times and so the data
+ * delete it, because it could be, that the server was down for some times and so the data
* are outdated
*/
if (file_exists($dataFilename) && (filemtime($dataFilename) < (time() - 5 * 60))) {
@@ -196,7 +207,7 @@
} else {
$data = array();
}
-
+
/* Thats it */
return $data;
}
@@ -218,18 +229,18 @@
}
/**
- * restarts apache, if needed
+ * restarts httpd, if needed
*/
- private function _rescueApache(){
+ private function _rescueHttpd(){
global $app, $conf;
-
+
/*
- * do nothing, if it is not allowed to rescue apache
+ * do nothing, if it is not allowed to rescue httpd
*/
- if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_apache']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_apache']) == 'y')){
+ if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_httpd']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_httpd']) == 'y')){
return;
}
-
+
/*
* if the service is up and running, or the service is not installed there is nothing to do...
*/
@@ -238,64 +249,147 @@
$this->_rescueData['webserver']['try_counter'] = 0;
return;
}
-
+
/*
* OK, the service is installed and down.
* Maybe this is because of a restart of the service by the admin.
* This means, we check the data 1 minute ago
*/
- if ((!isset($this->_monitoringData[1][0]['data']['webserver'])) ||
- ((isset($this->_monitoringData[1][0]['data']['webserver'])) && ($this->_monitoringData[1][0]['data']['webserver'] != 0))){
- /*
- * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago.
- * This means, it could be, that the admin is restarting the server.
+ if ((!isset($this->_monitoringData[1][0]['data']['webserver'])) ||
+ ((isset($this->_monitoringData[1][0]['data']['webserver'])) && ($this->_monitoringData[1][0]['data']['webserver'] != 0))){
+ /*
+ * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago.
+ * This means, it could be, that the admin is restarting the server.
* We wait one more minute...
*/
return;
}
-
+
/*#####
* The service is down and it was down 1 minute ago.
* We try to rescue it
*#####*/
-
+
/* Get the try counter */
$tryCount = (!isset($this->_rescueData['webserver']['try_counter']))? 1 : $this->_rescueData['webserver']['try_counter'] + 1;
-
+
/* Set the new try counter */
$this->_rescueData['webserver']['try_counter'] = $tryCount;
+ if ($tryCount > 2 && $conf['serverconfig']['web']['server_type'] != 'nginx') {
+ if($app->system->is_user('apache')) {
+ $app->log("Clearing semaphores table for user apache.",LOGLEVEL_WARN);
+ exec("ipcs -s | grep apache | awk '{ print $2 }' | xargs ipcrm sem");
+ }
+ if($app->system->is_user('www-data')) {
+ $app->log("Clearing semaphores table for user apache.",LOGLEVEL_WARN);
+ exec("ipcs -s | grep www-data | awk '{ print $2 }' | xargs ipcrm sem");
+ }
+ }
+
/* if 5 times will not work, we have to give up... */
if ($tryCount > 5){
- $app->log('Apache is down! Rescue will not help!', LOGLEVEL_ERROR);
+ $app->log('httpd is down! Rescue will not help!', LOGLEVEL_ERROR);
return;
}
-
-
- $app->log('Apache is down! Try rescue apache (try:' . $tryCount . ')...', LOGLEVEL_WARN);
- if(is_file($conf['init_scripts'] . '/' . 'httpd')) {
- $daemon = 'httpd';
+
+ $app->log('httpd is down! Try rescue httpd (try:' . $tryCount . ')...', LOGLEVEL_WARN);
+
+ if($conf['serverconfig']['web']['server_type'] == 'nginx'){
+ $daemon = 'nginx';
} else {
- $daemon = 'apache2';
+ if(is_file($conf['init_scripts'] . '/' . 'httpd')) {
+ $daemon = 'httpd';
+ } elseif(is_file($conf['init_scripts'] . '/' . 'httpd2')){
+ $daemon = 'httpd2';
+ } else {
+ $daemon = 'apache2';
+ }
}
-
+
$this->_rescueDaemon($daemon);
}
-
+
+ /**
+ * restarts MongoDB, if needed
+ */
+ private function _rescueMongoDB(){
+ global $app, $conf;
+
+ /*
+ * do nothing, if it is not allowed to rescue mysql
+ */
+ if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_mongodb']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_mongodb']) == 'y')){
+ return;
+ }
+
+ /*
+ * if the service is up and running, or the service is not installed there is nothing to do...
+ */
+ if ($this->_monitoringData[0][0]['data']['mongodbserver'] != 0){
+ /* Clear the try counter, because we do not have to try to rescue the service */
+ $this->_rescueData['mongodbserver']['try_counter'] = 0;
+ return;
+ }
+
+ /*
+ * OK, the service is installed and down.
+ * Maybe this is because of a restart of the service by the admin.
+ * This means, we check the data 1 minute ago
+ */
+ if ((!isset($this->_monitoringData[1][0]['data']['mongodbserver'])) ||
+ ((isset($this->_monitoringData[1][0]['data']['mongodbserver'])) && ($this->_monitoringData[1][0]['data']['mongodbserver'] != 0))){
+ /*
+ * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago.
+ * This means, it could be, that the admin is restarting the server.
+ * We wait one more minute...
+ */
+ return;
+ }
+
+ /*#####
+ * The service is down and it was down 1 minute ago.
+ * We try to rescue it
+ *#####*/
+
+ /* Get the try counter */
+ $tryCount = (!isset($this->_rescueData['mongodbserver']['try_counter']))? 1 : $this->_rescueData['mongodbserver']['try_counter'] + 1;
+
+ /* Set the new try counter */
+ $this->_rescueData['mongodbserver']['try_counter'] = $tryCount;
+
+ /* if 5 times will not work, we have to give up... */
+ if ($tryCount > 5){
+ $app->log('MongoDB is down! Rescue will not help!', LOGLEVEL_ERROR);
+ return;
+ }
+
+
+ $app->log('MongoDB is down! Try rescue MongoDB (try:' . $tryCount . ')...', LOGLEVEL_WARN);
+
+ if(is_file($conf['init_scripts'] . '/' . 'mongodb')) {
+ $daemon = 'mongodb';
+ } else {
+ $daemon = 'mongodb';
+ }
+
+ $this->_rescueDaemon($daemon);
+ }
+
/**
* restarts mysql, if needed
*/
private function _rescueMySql(){
global $app, $conf;
-
+
/*
* do nothing, if it is not allowed to rescue mysql
*/
if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) == 'y')){
return;
}
-
+
/*
* if the service is up and running, or the service is not installed there is nothing to do...
*/
@@ -304,40 +398,40 @@
$this->_rescueData['mysqlserver']['try_counter'] = 0;
return;
}
-
+
/*
* OK, the service is installed and down.
* Maybe this is because of a restart of the service by the admin.
* This means, we check the data 1 minute ago
*/
- if ((!isset($this->_monitoringData[1][0]['data']['mysqlserver'])) ||
- ((isset($this->_monitoringData[1][0]['data']['mysqlserver'])) && ($this->_monitoringData[1][0]['data']['mysqlserver'] != 0))){
- /*
- * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago.
- * This means, it could be, that the admin is restarting the server.
+ if ((!isset($this->_monitoringData[1][0]['data']['mysqlserver'])) ||
+ ((isset($this->_monitoringData[1][0]['data']['mysqlserver'])) && ($this->_monitoringData[1][0]['data']['mysqlserver'] != 0))){
+ /*
+ * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago.
+ * This means, it could be, that the admin is restarting the server.
* We wait one more minute...
*/
return;
}
-
+
/*#####
* The service is down and it was down 1 minute ago.
* We try to rescue it
*#####*/
-
+
/* Get the try counter */
$tryCount = (!isset($this->_rescueData['mysqlserver']['try_counter']))? 1 : $this->_rescueData['mysqlserver']['try_counter'] + 1;
-
+
/* Set the new try counter */
$this->_rescueData['mysqlserver']['try_counter'] = $tryCount;
-
+
/* if 5 times will not work, we have to give up... */
if ($tryCount > 5){
$app->log('MySQL is down! Rescue will not help!', LOGLEVEL_ERROR);
return;
}
-
-
+
+
$app->log('MySQL is down! Try rescue mysql (try:' . $tryCount . ')...', LOGLEVEL_WARN);
if(is_file($conf['init_scripts'] . '/' . 'mysqld')) {
@@ -345,41 +439,44 @@
} else {
$daemon = 'mysql';
}
-
+
$this->_rescueDaemon($daemon);
}
/**
* Tries to stop and then restart the daemon
- *
+ *
* @param type $daemon the name of the daemon
*/
private function _rescueDaemon($daemon){
- global $conf;
-
+ global $app, $conf;
+
+ $app->uses('system');
// if you need to find all restarts search for "['init_scripts']"
/*
* First we stop the running service "normally"
*/
-
+
/*
* ATTENTION!
* The service hangs. this means it could be, that "stop" will hang also.
* So we have to try to stop but if this will not work, we have to kill the stopping
* of the service
*/
- exec($conf['init_scripts'] . '/' . $daemon . ' stop && (sleep 3; kill $!; sleep 2; kill -9 $!) &> /dev/null');
-
+ exec($app->system->getinitcommand($daemon, 'stop').' && (sleep 3; kill $!; sleep 2; kill -9 $!) &> /dev/null');
+
/*
* OK, we tryed to stop it normally, maybe this worked maybe not. So we have to look
* if the service is already running or not. If so, we have to kill them hard
*/
exec("kill -9 `ps -A | grep " . $daemon . "| grep -v grep | awk '{print $1}'` &> /dev/null");
-
+
/*
* There are no more zombies left. Lets start the service..
*/
- exec($conf['init_scripts'] . '/' . $daemon . ' start');
+ exec($app->system->getinitcommand($daemon, 'start'));
}
+
}
+
?>
--
Gitblit v1.9.1