From eed6b3c6f189fcb318a38ef70c66d1501123f342 Mon Sep 17 00:00:00 2001
From: vogelor <vogelor@ispconfig3>
Date: Mon, 28 Mar 2011 15:53:05 -0400
Subject: [PATCH] the new rescue-module is now able to rescue (restart) mysql
---
server/lib/classes/monitor_tools.inc.php | 81 ++++++++++
server/lib/classes/db_mysql.inc.php | 10
server/mods-available/rescue_core_module.inc.php | 94 ++++++++++++
server/server.php | 194 ++++++++++++++++-----------
4 files changed, 280 insertions(+), 99 deletions(-)
diff --git a/server/lib/classes/db_mysql.inc.php b/server/lib/classes/db_mysql.inc.php
index 445563e..2c113b5 100644
--- a/server/lib/classes/db_mysql.inc.php
+++ b/server/lib/classes/db_mysql.inc.php
@@ -61,8 +61,8 @@
function updateError($location)
{
global $app;
- $this->errorNumber = mysql_errno($this->linkId);
- $this->errorMessage = mysql_error($this->linkId);
+ $this->errorNumber = @mysql_errno($this->linkId);
+ $this->errorMessage = @mysql_error($this->linkId);
$this->errorLocation = $location;
if($this->errorNumber && $this->show_error_messages && method_exists($app,'log'))
{
@@ -76,7 +76,7 @@
{
if($this->linkId == 0)
{
- $this->linkId = mysql_connect($this->dbHost, $this->dbUser, $this->dbPass);
+ $this->linkId = @mysql_connect($this->dbHost, $this->dbUser, $this->dbPass);
if(!$this->linkId)
{
$this->updateError('DB::connect()-> mysql_connect');
@@ -463,7 +463,7 @@
return $this->query($sql);
}
- // gibt Array mit Tabellennamen zur�ck
+ // gibt Array mit Tabellennamen zur�ck
function getTables($database_name = '') {
if($database_name == '') $database_name = $this->dbName;
@@ -474,7 +474,7 @@
return $tb_names;
}
- // gibt Feldinformationen zur Tabelle zur�ck
+ // gibt Feldinformationen zur Tabelle zur�ck
/*
$columns = array(action => add | alter | drop
name => Spaltenname
diff --git a/server/lib/classes/monitor_tools.inc.php b/server/lib/classes/monitor_tools.inc.php
index 2fbd402..3ac8593 100644
--- a/server/lib/classes/monitor_tools.inc.php
+++ b/server/lib/classes/monitor_tools.inc.php
@@ -518,8 +518,16 @@
/** the id of the server as int */
$server_id = intval($conf['server_id']);
- /** get the "active" Services of the server from the DB */
+ /** get the "active" Services of the server from the DB */
$services = $app->dbmaster->queryOneRecord('SELECT * FROM server WHERE server_id = ' . $server_id);
+ /*
+ * If the DB is down, we have to set the db to "yes".
+ * If we don't do this, then the monitor will NOT monitor, that the db is down and so the
+ * rescue-module can not try to rescue the db
+ */
+ if ($services == null) {
+ $services['db_server'] = 1;
+ }
/* The type of the Monitor-data */
$type = 'services';
@@ -1539,16 +1547,16 @@
* We got a connection, but maybe apache is not able to send data over this
* connection?
*/
- fwrite($fp, "GET / HTTP/1.0\r\n\r\n");
+ fwrite($fp, "GET / HTTP/1.0\r\n\r\n");
stream_set_timeout($fp, 2);
$res = fread($fp, 10);
- $info = stream_get_meta_data($fp);
+ $info = stream_get_meta_data($fp);
fclose($fp);
- if ($info['timed_out']) {
+ if ($info['timed_out']) {
return false; // Apache was not able to send data over this connection
- } else {
+ } else {
return true; // Apache was able to send data over this connection
- }
+ }
} else {
return false; // Apache was not able to establish a connection
}
@@ -1577,6 +1585,67 @@
return false;
}
}
+
+ /*
+ * Set the state to the given level (or higher, but not lesser).
+ * * If the actual state is critical and you call the method with ok,
+ * then the state is critical.
+ *
+ * * If the actual state is critical and you call the method with error,
+ * then the state is error.
+ */
+ private function _setState($oldState, $newState)
+ {
+ /*
+ * Calculate the weight of the old state
+ */
+ switch ($oldState) {
+ case 'no_state': $oldInt = 0;
+ break;
+ case 'ok': $oldInt = 1;
+ break;
+ case 'unknown': $oldInt = 2;
+ break;
+ case 'info': $oldInt = 3;
+ break;
+ case 'warning': $oldInt = 4;
+ break;
+ case 'critical': $oldInt = 5;
+ break;
+ case 'error': $oldInt = 6;
+ break;
+ }
+ /*
+ * Calculate the weight of the new state
+ */
+ switch ($newState) {
+ case 'no_state': $newInt = 0 ;
+ break;
+ case 'ok': $newInt = 1 ;
+ break;
+ case 'unknown': $newInt = 2 ;
+ break;
+ case 'info': $newInt = 3 ;
+ break;
+ case 'warning': $newInt = 4 ;
+ break;
+ case 'critical': $newInt = 5 ;
+ break;
+ case 'error': $newInt = 6 ;
+ break;
+ }
+
+ /*
+ * Set to the higher level
+ */
+ if ($newInt > $oldInt){
+ return $newState;
+ }
+ else
+ {
+ return $oldState;
+ }
+ }
private function _getIntArray($line) {
/** The array of float found */
diff --git a/server/mods-available/rescue_core_module.inc.php b/server/mods-available/rescue_core_module.inc.php
index 7ce828d..4fcd60c 100644
--- a/server/mods-available/rescue_core_module.inc.php
+++ b/server/mods-available/rescue_core_module.inc.php
@@ -94,6 +94,11 @@
$this->_rescueApache();
/*
+ * rescue mysql if needed
+ */
+ $this->_rescueMySql();
+
+ /*
* The last step is to save the rescue-data
*/
$this->_saveRescueData();
@@ -263,15 +268,94 @@
$app->log('Apache is down! Try rescue apache (try:' . $tryCount . ')...', LOGLEVEL_WARN);
// echo 'Apache is down! Try rescue apache (try:' . $tryCount . ')...';
- /*
- * First we stop the running service "normally"
- */
- $daemon = '';
if(is_file($conf['init_scripts'] . '/' . 'httpd')) {
$daemon = 'httpd';
} else {
$daemon = 'apache2';
}
+
+ $this->_rescueDaemon($daemon);
+ }
+
+ /**
+ * restarts mysql, if needed
+ */
+ private function _rescueMySql(){
+ global $app, $conf;
+
+ /*
+ * do nothing, if it is not allowed to rescue mysql
+ */
+ if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) == 'y')){
+ return;
+ }
+
+ /*
+ * if the service is up and running, or the service is not installed there is nothing to do...
+ */
+ if ($this->_monitoringData[0][0]['data']['mysqlserver'] != 0){
+ /* Clear the try counter, because we do not have to try to rescue the service */
+ $this->_rescueData['mysqlserver']['try_counter'] = 0;
+ return;
+ }
+
+ /*
+ * OK, the service is installed and down.
+ * Maybe this is because of a restart of the service by the admin.
+ * This means, we check the data 1 minute ago
+ */
+ if ((!isset($this->_monitoringData[1][0]['data']['mysqlserver'])) ||
+ ((isset($this->_monitoringData[1][0]['data']['mysqlserver'])) && ($this->_monitoringData[1][0]['data']['mysqlserver'] != 0))){
+ /*
+ * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago.
+ * This means, it could be, that the admin is restarting the server.
+ * We wait one more minute...
+ */
+ return;
+ }
+
+ /*#####
+ * The service is down and it was down 1 minute ago.
+ * We try to rescue it
+ *#####*/
+
+ /* Get the try counter */
+ $tryCount = (!isset($this->_rescueData['mysqlserver']['try_counter']))? 1 : $this->_rescueData['mysqlserver']['try_counter'] + 1;
+
+ /* Set the new try counter */
+ $this->_rescueData['mysqlserver']['try_counter'] = $tryCount;
+
+ /* if 5 times will not work, we have to give up... */
+ if ($tryCount > 5){
+ $app->log('MySQL is down! Rescue will not help!', LOGLEVEL_ERROR);
+ return;
+ }
+
+
+ $app->log('MySQL is down! Try rescue mysql (try:' . $tryCount . ')...', LOGLEVEL_WARN);
+// echo 'MySQL is down! Try rescue mysql (try:' . $tryCount . ')...';
+
+ if(is_file($conf['init_scripts'] . '/' . 'mysqld')) {
+ $daemon = 'mysqld';
+ } else {
+ $daemon = 'mysql';
+ }
+
+ $this->_rescueDaemon($daemon);
+ }
+
+ /**
+ * Tries to stop and then restart the daemon
+ *
+ * @param type $daemon the name of the daemon
+ */
+ private function _rescueDaemon($daemon){
+ global $conf;
+
+ // if you need to find all restarts search for "['init_scripts']"
+ /*
+ * First we stop the running service "normally"
+ */
/*
* ATTENTION!
@@ -292,7 +376,5 @@
*/
exec($conf['init_scripts'] . '/' . $daemon . ' start');
}
-
-// if you need to find all restarts search for "['init_scripts']"
}
?>
diff --git a/server/server.php b/server/server.php
index 537176c..6a3499f 100644
--- a/server/server.php
+++ b/server/server.php
@@ -1,110 +1,132 @@
<?php
/*
-Copyright (c) 2007, Till Brehm, projektfarm Gmbh
-All rights reserved.
+ Copyright (c) 2007, Till Brehm, projektfarm Gmbh
+ All rights reserved.
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- * Neither the name of ISPConfig nor the names of its contributors
- may be used to endorse or promote products derived from this software without
- specific prior written permission.
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of ISPConfig nor the names of its contributors
+ may be used to endorse or promote products derived from this software without
+ specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
-INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
-EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
require('lib/config.inc.php');
require('lib/app.inc.php');
set_time_limit(0);
-ini_set('error_reporting',E_ALL & ~E_NOTICE);
+ini_set('error_reporting', E_ALL & ~E_NOTICE);
// make sure server_id is always an int
$conf['server_id'] = intval($conf['server_id']);
/*
-// Get server record, if updates where available for this server
-$server_db_record = $app->db->queryOneRecord("SELECT * FROM server WHERE update = 1 AND server_id = ".$conf['server_id']);
-if($server_db_record == false) {
- $app->log('Nothing to update for server_id '.$conf['server_id']);
- die();
-} else {
- // Set update status to 0, so we dont start the update process twice
- $app->db->query("UPDATE server SET update = 0 WHERE server_id = ".$conf['server_id']);
- $app->log('Begin update.');
-}
-*/
+ // Get server record, if updates where available for this server
+ $server_db_record = $app->db->queryOneRecord("SELECT * FROM server WHERE update = 1 AND server_id = ".$conf['server_id']);
+ if($server_db_record == false) {
+ $app->log('Nothing to update for server_id '.$conf['server_id']);
+ die();
+ } else {
+ // Set update status to 0, so we dont start the update process twice
+ $app->db->query("UPDATE server SET update = 0 WHERE server_id = ".$conf['server_id']);
+ $app->log('Begin update.');
+ }
+ */
-//* Load the server configuration
-if($app->dbmaster->connect()) {
+/*
+ * Try to Load the server configuration from the master-db
+ */
+if ($app->dbmaster->connect()) {
// get the dalaog_id of the last performed record
- $server_db_record = $app->dbmaster->queryOneRecord("SELECT * FROM server WHERE server_id = ".$conf['server_id']);
- $conf['last_datalog_id'] = (int)$server_db_record['updated'];
- $conf['mirror_server_id'] = (int)$server_db_record['mirror_server_id'];
+ $server_db_record = $app->dbmaster->queryOneRecord("SELECT * FROM server WHERE server_id = " . $conf['server_id']);
+ $conf['last_datalog_id'] = (int) $server_db_record['updated'];
+ $conf['mirror_server_id'] = (int) $server_db_record['mirror_server_id'];
// Load the ini_parser
$app->uses('ini_parser');
// Get server configuration
$conf['serverconfig'] = $app->ini_parser->parse_ini_string(stripslashes($server_db_record['config']));
// Set the loglevel
$conf['log_priority'] = intval($conf['serverconfig']['server']['loglevel']);
-
+
unset($server_db_record);
+} else {
+ /*
+ * The master-db is not available.
+ * Problem: because we need to start the rescue-module (to rescue the DB if this IS the
+ * server, the master-db is running at) we have to initialize some config...
+ */
+ $conf['last_datalog_id'] = intval('9223372036854775807'); // maxint at 32 and 64 bit systems
+ $conf['mirror_server_id'] = 0; // no mirror
+ // Set the loglevel to warning
+ $conf['log_priority'] = LOGLEVEL_WARN;
+ /*
+ * Set the configuration to rescue the database
+ */
+ $conf['serverconfig']['rescue']['try_rescue'] = 'y';
+ $conf['serverconfig']['rescue']['do_not_try_rescue_mysql'] = 'n';
}
// Check whether another instance of this script is already running
-if(is_file($conf['temppath'].$conf['fs_div'].'.ispconfig_lock')){
- clearstatcache();
- for($i=0;$i<120;$i++){ // Wait max. 1200 sec, then retry
- if(is_file($conf['temppath'].$conf['fs_div'].'.ispconfig_lock')){
- exec("ps aux | grep '/usr/local/ispconfig/server/[s]erver.php' | wc -l", $check);
- if(intval($check[0]) > 1) { // 1 because this is 2nd instance!
- $app->log('There is already an instance of server.php running. Exiting.', LOGLEVEL_DEBUG);
- exit;
- }
- $app->log('There is already a lockfile set. Waiting another 10 seconds...', LOGLEVEL_DEBUG);
- sleep(10);
- clearstatcache();
- }
- }
+if (is_file($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock')) {
+ clearstatcache();
+ for ($i = 0; $i < 120; $i++) { // Wait max. 1200 sec, then retry
+ if (is_file($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock')) {
+ exec("ps aux | grep '/usr/local/ispconfig/server/[s]erver.php' | wc -l", $check);
+ if (intval($check[0]) > 1) { // 1 because this is 2nd instance!
+ $app->log('There is already an instance of server.php running. Exiting.', LOGLEVEL_DEBUG);
+ exit;
+ }
+ $app->log('There is already a lockfile set. Waiting another 10 seconds...', LOGLEVEL_DEBUG);
+ sleep(10);
+ clearstatcache();
+ }
+ }
}
// Set Lockfile
-@touch($conf['temppath'].$conf['fs_div'].'.ispconfig_lock');
-$app->log('Set Lock: '.$conf['temppath'].$conf['fs_div'].'.ispconfig_lock', LOGLEVEL_DEBUG);
+@touch($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock');
+$app->log('Set Lock: ' . $conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock', LOGLEVEL_DEBUG);
+/** Do we need to start the core-modules */
+$needStartCore = true;
-if($app->db->connect() && $app->dbmaster->connect()) {
+/*
+ * Next we try to process the datalog
+ */
+if ($app->db->connect() && $app->dbmaster->connect()) {
// Check if there is anything to update
- if($conf['mirror_server_id'] > 0) {
- $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > ".$conf['last_datalog_id']." AND (server_id = ".$conf['server_id']." OR server_id = ".$conf['mirror_server_id']." OR server_id = 0)");
+ if ($conf['mirror_server_id'] > 0) {
+ $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > " . $conf['last_datalog_id'] . " AND (server_id = " . $conf['server_id'] . " OR server_id = " . $conf['mirror_server_id'] . " OR server_id = 0)");
} else {
- $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > ".$conf['last_datalog_id']." AND (server_id = ".$conf['server_id']." OR server_id = 0)");
+ $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > " . $conf['last_datalog_id'] . " AND (server_id = " . $conf['server_id'] . " OR server_id = 0)");
}
-
+
$tmp_num_records = $tmp_rec['number'];
unset($tmp_rec);
- if($tmp_num_records > 0) {
+ if ($tmp_num_records > 0) {
/*
- There is something to do, triggert by the database -> do it!
- */
+ There is something to do, triggert by the database -> do it!
+ */
// Write the Log
$app->log("Found $tmp_num_records changes, starting update process.", LOGLEVEL_DEBUG);
// Load required base-classes
@@ -118,30 +140,38 @@
$app->modules->processDatalog();
// Restart services that need to after configuration
$app->services->processDelayedActions();
- } else {
- /*
- There is no trigger inside the database -> load only the core, maybe they have to do something
- */
- // Write the log
- $app->log('No Updated records found, starting only the core.', LOGLEVEL_DEBUG);
- // Load required base-classes
- $app->uses('modules,plugins,file,services');
- // Load the modules that are im the mods-core folder
- $app->modules->loadModules('core');
- // Load the plugins that are in the plugins-core folder
- $app->plugins->loadPlugins('core');
+ // All modules are already loaded and processed, so there is NO NEED to load the core once again...
+ $needStartCore = false;
}
} else {
- if(!$app->db->connect()) {
- $app->log('Unable to connect to local server.'.$app->db->errorMessage,LOGLEVEL_WARN);
+ if (!$app->db->connect()) {
+ $app->log('Unable to connect to local server.' . $app->db->errorMessage, LOGLEVEL_WARN);
} else {
- $app->log('Unable to connect to master server.'.$app->dbmaster->errorMessage,LOGLEVEL_WARN);
+ $app->log('Unable to connect to master server.' . $app->dbmaster->errorMessage, LOGLEVEL_WARN);
}
}
+/*
+ * Under normal circumstances the system was loaded and all updates are done.
+ * but if we do not have to update anything or if the database is not accessible, then we
+ * have to start the core-system (if the database is accessible, we need the core because of the
+ * monitoring. If the databse is NOT accessible, we need the core because of rescue the db...
+ */
+if ($needStartCore) {
+ // Write the log
+ $app->log('No Updated records found, starting only the core.', LOGLEVEL_DEBUG);
+ // Load required base-classes
+ $app->uses('modules,plugins,file,services');
+ // Load the modules that are im the mods-core folder
+ $app->modules->loadModules('core');
+ // Load the plugins that are in the plugins-core folder
+ $app->plugins->loadPlugins('core');
+}
+
+
// Remove lock
-@unlink($conf['temppath'].$conf['fs_div'].'.ispconfig_lock');
-$app->log('Remove Lock: '.$conf['temppath'].$conf['fs_div'].'.ispconfig_lock',LOGLEVEL_DEBUG);
+@unlink($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock');
+$app->log('Remove Lock: ' . $conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock', LOGLEVEL_DEBUG);
die("finished.\n");
--
Gitblit v1.9.1