| commit | author | age | ||
| 146783 | 1 | <?php |
| MC | 2 | /* |
| 3 | Copyright (c) 2012, ISPConfig UG | |
| 4 | Contributors: web wack creations, http://www.web-wack.at | |
| 5 | All rights reserved. | |
| 6 | ||
| 7 | Redistribution and use in source and binary forms, with or without modification, | |
| 8 | are permitted provided that the following conditions are met: | |
| 9 | ||
| 10 | * Redistributions of source code must retain the above copyright notice, | |
| 11 | this list of conditions and the following disclaimer. | |
| 12 | * Redistributions in binary form must reproduce the above copyright notice, | |
| 13 | this list of conditions and the following disclaimer in the documentation | |
| 14 | and/or other materials provided with the distribution. | |
| 15 | * Neither the name of ISPConfig nor the names of its contributors | |
| 16 | may be used to endorse or promote products derived from this software without | |
| 17 | specific prior written permission. | |
| 18 | ||
| 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND | |
| 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | |
| 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | |
| 22 | IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, | |
| 23 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, | |
| 24 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY | |
| 26 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING | |
| 27 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, | |
| 28 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 29 | */ | |
| 7fe908 | 30 | require_once 'aps_base.inc.php'; |
| 146783 | 31 | |
| MC | 32 | @set_time_limit(0); |
| 33 | @ignore_user_abort(1); | |
| 34 | ||
| 35 | class ApsCrawler extends ApsBase | |
| 36 | { | |
| 37 | ||
| 7fe908 | 38 | //public $app_download_url_list = array(); |
| 146783 | 39 | |
| 7fe908 | 40 | /** |
| MC | 41 | * Constructor |
| 42 | * | |
| 43 | * @param $app the application instance (db handle + log method) | |
| 44 | * @param $interface_mode act in interface (true) or server mode (false) | |
| 45 | */ | |
| 146783 | 46 | |
| MC | 47 | |
| 7fe908 | 48 | public function __construct($app, $interface_mode = false) |
| MC | 49 | { |
| 50 | parent::__construct($app, 'APS crawler: ', $interface_mode); | |
| 51 | } | |
| 146783 | 52 | |
| MC | 53 | |
| 54 | ||
| 7fe908 | 55 | /** |
| MC | 56 | * Before the cron is executed, make sure all necessary options are set |
| 57 | * and all functions (i.e. cURL) are available | |
| 58 | */ | |
| 59 | private function checkRequirements() | |
| 60 | { | |
| 61 | global $app; | |
| 146783 | 62 | |
| 7fe908 | 63 | try |
| MC | 64 | { |
| 65 | // Check if allow_url_fopen is enabled | |
| 66 | if(!@ini_get('allow_url_fopen')) throw new Exception('allow_url_fopen is not enabled'); | |
| 67 | // Check if the cURL module is available | |
| 68 | if(!function_exists('curl_version')) throw new Exception('cURL is not available'); | |
| 69 | ||
| 70 | // Check if used folders are writable | |
| 71 | if($this->interface_mode) | |
| 72 | { | |
| 73 | if(!is_writable($this->interface_pkg_dir)) | |
| 74 | throw new Exception('the folder '.basename($this->interface_pkg_dir).' is not writable'); | |
| 75 | } | |
| 76 | else | |
| 77 | { | |
| 78 | if(!is_writable($this->packages_dir)) | |
| 79 | throw new Exception('the folder '.basename($this->packages_dir).' is not writable'); | |
| 80 | } | |
| 81 | ||
| 82 | return true; | |
| 83 | } | |
| 84 | ||
| 85 | catch(Exception $e) | |
| 86 | { | |
| 87 | $app->log($this->log_prefix.'Aborting execution because '.$e->getMessage(), LOGLEVEL_ERROR); | |
| 88 | return false; | |
| 89 | } | |
| 90 | } | |
| 91 | ||
| 92 | ||
| 93 | ||
| 94 | /** | |
| 95 | * Remove a directory recursively | |
| 96 | * In case of error be silent | |
| 97 | * | |
| 98 | * @param $dir the directory to remove | |
| 99 | */ | |
| 100 | private function removeDirectory($dir) | |
| 101 | { | |
| 102 | if(is_dir($dir)) | |
| 103 | { | |
| 104 | $files = scandir($dir); | |
| 105 | foreach($files as $file) | |
| 106 | { | |
| 107 | if($file != '.' && $file != '..') | |
| 108 | if(filetype($dir.'/'.$file) == 'dir') $this->removeDirectory($dir.'/'.$file); | |
| 109 | else @unlink($dir.'/'.$file); | |
| 110 | } | |
| 111 | reset($files); | |
| 112 | @rmdir($dir); | |
| 113 | } | |
| 114 | } | |
| 115 | ||
| 116 | ||
| 117 | /** | |
| 118 | * Fetch HTML data from one or more given URLs | |
| 119 | * If a string is given, a string is returned, if an array of URLs should | |
| 120 | * be fetched, the responses of the parallel queries are returned as array | |
| 121 | * | |
| 122 | * @param $input the string or array to fetch | |
| 123 | * @return $ret a query response string or array | |
| 124 | */ | |
| 125 | private function fetchPage($input) | |
| 126 | { | |
| 127 | $ret = array(); | |
| 128 | $url = array(); | |
| 129 | $conn = array(); | |
| 130 | ||
| 131 | // Make sure we are working with an array, further on | |
| 132 | if(!is_array($input)) $url[] = $input; | |
| 133 | else $url = $input; | |
| 134 | ||
| 135 | // Build the single cURL handles and add them to a multi handle | |
| 136 | $mh = curl_multi_init(); | |
| 137 | for($i = 0; $i < count($url); $i++) | |
| 138 | { | |
| 139 | $conn[$i] = curl_init('http://'.$this->fetch_url.$url[$i]); | |
| 140 | curl_setopt($conn[$i], CURLOPT_RETURNTRANSFER, true); | |
| 141 | curl_multi_add_handle($mh, $conn[$i]); | |
| 142 | } | |
| 143 | ||
| 144 | $active = 0; | |
| 145 | do curl_multi_exec($mh, $active); | |
| 146 | while($active > 0); | |
| 147 | ||
| 148 | // Get the response(s) | |
| 149 | for($i = 0; $i < count($url); $i++) | |
| 150 | { | |
| 151 | $ret[$i] = curl_multi_getcontent($conn[$i]); | |
| 152 | curl_multi_remove_handle($mh, $conn[$i]); | |
| 153 | curl_close($conn[$i]); | |
| 154 | } | |
| 155 | curl_multi_close($mh); | |
| 156 | ||
| 157 | if(count($url) == 1) $ret = $ret[0]; | |
| 158 | ||
| 159 | return $ret; | |
| 160 | } | |
| 161 | ||
| 162 | ||
| 163 | ||
| 164 | /** | |
| 165 | * Fetch binary data from a given array | |
| 166 | * The data is retrieved in binary mode and | |
| 167 | * then directly written to an output file | |
| 168 | * | |
| 169 | * @param $input a specially structed array | |
| 170 | * @see $this->startUpdate() | |
| 171 | */ | |
| 172 | private function fetchFiles($input) | |
| 173 | { | |
| 174 | $fh = array(); | |
| 175 | $url = array(); | |
| 176 | $conn = array(); | |
| 177 | ||
| 178 | // Build the single cURL handles and add them to a multi handle | |
| 179 | $mh = curl_multi_init(); | |
| 180 | ||
| 181 | // Process each app | |
| 182 | for($i = 0; $i < count($input); $i++) | |
| 183 | { | |
| 184 | $conn[$i] = curl_init($input[$i]['url']); | |
| 185 | $fh[$i] = fopen($input[$i]['localtarget'], 'wb'); | |
| 186 | ||
| 187 | curl_setopt($conn[$i], CURLOPT_BINARYTRANSFER, true); | |
| 188 | curl_setopt($conn[$i], CURLOPT_FILE, $fh[$i]); | |
| 189 | curl_setopt($conn[$i], CURLOPT_TIMEOUT, 0); | |
| 190 | curl_setopt($conn[$i], CURLOPT_FAILONERROR, 1); | |
| 191 | curl_setopt($conn[$i], CURLOPT_FOLLOWLOCATION, 1); | |
| 42f0c9 | 192 | curl_setopt($conn[$i], CURLOPT_SSL_VERIFYHOST, 1); |
| TB | 193 | curl_setopt($conn[$i], CURLOPT_SSL_VERIFYPEER, false); |
| 7fe908 | 194 | |
| MC | 195 | curl_multi_add_handle($mh, $conn[$i]); |
| 196 | } | |
| 197 | ||
| 198 | $active = 0; | |
| 199 | do curl_multi_exec($mh, $active); | |
| 200 | while($active > 0); | |
| 201 | ||
| 202 | // Close the handles | |
| 203 | for($i = 0; $i < count($input); $i++) | |
| 204 | { | |
| 205 | fclose($fh[$i]); | |
| 206 | curl_multi_remove_handle($mh, $conn[$i]); | |
| 207 | curl_close($conn[$i]); | |
| 208 | } | |
| 209 | curl_multi_close($mh); | |
| 210 | } | |
| 211 | ||
| 212 | ||
| 213 | ||
| 214 | /** | |
| 215 | * A method to build query URLs out of a list of vendors | |
| 216 | * | |
| 217 | */ | |
| 218 | private function formatVendorCallback($array_item) | |
| 219 | { | |
| 220 | $array_item = str_replace(' ', '%20', $array_item); | |
| 221 | $array_item = str_replace('http://', '', $array_item); | |
| 222 | $array_item = '/'.$this->aps_version.'.atom?vendor='.$array_item.'&pageSize=100'; | |
| 223 | return $array_item; | |
| 224 | } | |
| 225 | ||
| 226 | ||
| 227 | ||
| 228 | /** | |
| 229 | * The main method which performs the actual crawling | |
| 230 | */ | |
| 231 | public function startCrawler() | |
| 232 | { | |
| 233 | global $app; | |
| 234 | ||
| 235 | try | |
| 236 | { | |
| 237 | // Make sure the requirements are given so that this script can execute | |
| 238 | $req_ret = $this->checkRequirements(); | |
| 239 | if(!$req_ret) return false; | |
| 240 | ||
| 241 | // Execute the open task and first fetch all vendors (APS catalog API 1.1, p. 12) | |
| 242 | $app->log($this->log_prefix.'Fetching data from '.$this->fetch_url); | |
| 243 | ||
| 244 | $vendor_page = $this->fetchPage('/all-app/'); //$vendor_page = $this->fetchPage('/'.$this->aps_version.'/'); | |
| 245 | preg_match_all("/\<a href=\"(.+)\/\" class=\"vendor\"/i", $vendor_page, $matches); | |
| 246 | $vendors = array_map('urldecode', $matches[1]); | |
| 247 | if(!$vendors) throw new Exception('Unable to fetch vendors. Aborting'); | |
| 248 | ||
| 249 | // Format all vendors for further processing (i.e. typo3.org -> /1.atom?vendor=typo3.org&pageSize=100 | |
| 250 | //array_walk($vendors, array($this, 'formatVendorCallback')); | |
| 146783 | 251 | if(is_array($vendors)) { |
| MC | 252 | foreach($vendors as $key => $array_item) { |
| 253 | $vendors[$key] = $this->formatVendorCallback($array_item); | |
| 254 | } | |
| 255 | } | |
| 256 | ||
| 7fe908 | 257 | // Process all vendors in chunks of 50 entries |
| MC | 258 | $vendor_chunks = array_chunk($vendors, 50); |
| 259 | //var_dump($vendor_chunks); | |
| 260 | ||
| 261 | // Get all known apps from the database and the highest known version | |
| 262 | // Note: A dirty hack is used for numerical sorting of the VARCHAR field Version: +0 -> cast | |
| 263 | // A longer but typesafe way would be: ORDER BY CAST(REPLACE(Version, '.', '') AS UNSIGNED) DESC | |
| 264 | $existing_apps = $app->db->queryAllRecords("SELECT * FROM ( | |
| 265 | SELECT name AS Name, CONCAT(version, '-', CAST(`release` AS CHAR)) AS CurrentVersion | |
| 146783 | 266 | FROM aps_packages ORDER BY REPLACE(version, '.', '')+0 DESC, `release` DESC |
| MC | 267 | ) as Versions GROUP BY name"); |
| 7fe908 | 268 | //var_dump($existing_apps); |
| MC | 269 | |
| 270 | // Used for statistics later | |
| 271 | $apps_in_repo = 0; | |
| 272 | $apps_updated = 0; | |
| 273 | $apps_downloaded = 0; | |
| 274 | ||
| 275 | $apps_to_dl = array(); | |
| 276 | ||
| 277 | for($i = 0; $i < count($vendor_chunks); $i++) | |
| 278 | { | |
| 279 | // Fetch all apps for the current chunk of vendors | |
| 280 | $apps = $this->fetchPage($vendor_chunks[$i]); | |
| 281 | ||
| 282 | for($j = 0; $j < count($apps); $j++) | |
| 283 | { | |
| 284 | // Before parsing, make sure it's worth the work by checking if at least one app exists | |
| 285 | $apps_count = substr_count($apps[$j], '<opensearch:totalResults>0</opensearch:totalResults>'); | |
| 286 | if($apps_count == 0) // obviously this vendor provides one or more apps | |
| 287 | { | |
| 42f0c9 | 288 | try { |
| 7fe908 | 289 | // Rename namespaces and register them |
| MC | 290 | $xml = str_replace("xmlns=", "ns=", $apps[$j]); |
| 291 | $sxe = new SimpleXMLElement($xml); | |
| 292 | $namespaces = $sxe->getDocNamespaces(true); | |
| 293 | foreach($namespaces as $ns => $url) $sxe->registerXPathNamespace($ns, $url); | |
| 54d081 | 294 | |
| TB | 295 | //Find highest version |
| 296 | $app_version = "0.0.0"; | |
| 297 | $entry_pos = 1; | |
| 298 | for ($p = 1; ; $p++) { | |
| 299 | $app_version_tmp = parent::getXPathValue($sxe, 'entry[position()=' . $p . ']/a:version'); | |
| 300 | if (strlen($app_version_tmp) < 1) break; | |
| 301 | if (version_compare($app_version_tmp, $app_version) >= 0) { | |
| 302 | $app_version = $app_version_tmp; | |
| 303 | $entry_pos = $p; | |
| 304 | } | |
| 305 | } | |
| 7fe908 | 306 | |
| MC | 307 | // Fetching values of interest |
| 54d081 | 308 | //$app_name = parent::getXPathValue($sxe, 'entry[position()=1]/a:name'); |
| TB | 309 | //$app_version = parent::getXPathValue($sxe, 'entry[position()=1]/a:version'); |
| 310 | //$app_release = parent::getXPathValue($sxe, 'entry[position()=1]/a:release'); | |
| 311 | $app_name = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:name"); | |
| 312 | $app_version = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:version"); | |
| 313 | $app_release = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/a:release"); | |
| 7fe908 | 314 | |
| MC | 315 | // Find out a (possibly) existing package version |
| 316 | $ex_ver = ''; | |
| 146783 | 317 | /* |
| 7fe908 | 318 | array_walk($existing_apps, |
| 146783 | 319 | create_function('$v, $k, $ex_ver', 'if($v["Name"] == "'.$app_name.'") $ex_ver = $v["CurrentVersion"];'), &$ex_ver); |
| MC | 320 | */ |
| 321 | if(is_array($existing_apps)) { | |
| 322 | foreach($existing_apps as $k => $v) { | |
| 323 | if($v["Name"] == $app_name) $ex_ver = $v["CurrentVersion"]; | |
| 324 | } | |
| 325 | } | |
| 326 | ||
| 7fe908 | 327 | $new_ver = $app_version.'-'.$app_release; |
| MC | 328 | $local_intf_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$new_ver.'.app.zip/'; |
| 146783 | 329 | |
| 7fe908 | 330 | // Proceed if a newer or at least equal version has been found with server mode or |
| MC | 331 | // interface mode is activated and there are no valid APP-META.xml and PKG_URL existing yet |
| 332 | if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0) || ($this->interface_mode && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0 || !file_exists($local_intf_folder.'PKG_URL') || filesize($local_intf_folder.'PKG_URL') == 0))){ | |
| 333 | // Check if we already have an old version of this app | |
| 334 | if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) $apps_updated++; | |
| 335 | ||
| 54d081 | 336 | //$app_dl = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@href"); |
| TB | 337 | //$app_filesize = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@length"); |
| 338 | //$app_metafile = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='meta']/@href"); | |
| 339 | $app_dl = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='aps']/@href"); | |
| 340 | $app_filesize = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='aps']/@length"); | |
| 341 | $app_metafile = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='meta']/@href"); | |
| 7fe908 | 342 | |
| 146783 | 343 | //$this->app_download_url_list[$app_name.'-'.$new_ver.'.app.zip'] = $app_dl; |
| 7fe908 | 344 | // Skip ASP.net packages because they can't be used at all |
| MC | 345 | $asp_handler = parent::getXPathValue($sxe, '//aspnet:handler'); |
| 346 | $asp_permissions = parent::getXPathValue($sxe, '//aspnet:permissions'); | |
| 347 | $asp_version = parent::getXPathValue($sxe, '//aspnet:version'); | |
| 348 | if(!empty($asp_handler) || !empty($asp_permissions) || !empty($asp_version)) continue; | |
| 146783 | 349 | |
| 7fe908 | 350 | // Interface mode (download only parts) |
| MC | 351 | if($this->interface_mode) |
| 352 | { | |
| 353 | // Delete an obviously out-dated version from the system and DB | |
| 354 | if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) | |
| 355 | { | |
| 356 | $old_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; | |
| 357 | if(file_exists($old_folder)) $this->removeDirectory($old_folder); | |
| 358 | ||
| 146783 | 359 | /* |
| MC | 360 | $app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '". |
| 361 | $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". | |
| 362 | $app->db->quote($ex_ver)."';"); | |
| 363 | */ | |
| 364 | $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '". | |
| 7fe908 | 365 | $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". |
| MC | 366 | $app->db->quote($ex_ver)."';"); |
| 146783 | 367 | $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']); |
| MC | 368 | unset($tmp); |
| 7fe908 | 369 | } |
| MC | 370 | |
| 371 | // Create the local folder if not yet existing | |
| 372 | if(!file_exists($local_intf_folder)) @mkdir($local_intf_folder, 0777, true); | |
| 373 | ||
| 146783 | 374 | // Save the package URL in an extra file because it's not part of the APP-META.xml file |
| MC | 375 | @file_put_contents($local_intf_folder.'PKG_URL', $app_dl); |
| 376 | ||
| 7fe908 | 377 | // Download the meta file |
| MC | 378 | $local_metafile = $local_intf_folder.'APP-META.xml'; |
| 379 | if(!file_exists($local_metafile) || filesize($local_metafile) == 0) | |
| 380 | { | |
| 381 | $apps_to_dl[] = array('name' => 'APP-META.xml', | |
| 382 | 'url' => $app_metafile, | |
| 383 | 'filesize' => 0, | |
| 384 | 'localtarget' => $local_metafile); | |
| 385 | $apps_downloaded++; | |
| 386 | } | |
| 387 | ||
| 388 | // Download package license | |
| 54d081 | 389 | //$license = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='eula']/@href"); |
| TB | 390 | $license = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='eula']/@href"); |
| 7fe908 | 391 | if($license != '') |
| MC | 392 | { |
| 393 | $local_license = $local_intf_folder.'LICENSE'; | |
| 394 | if(!file_exists($local_license) || filesize($local_license) == 0) | |
| 395 | { | |
| 396 | $apps_to_dl[] = array('name' => basename($license), | |
| 397 | 'url' => $license, | |
| 398 | 'filesize' => 0, | |
| 399 | 'localtarget' => $local_license); | |
| 400 | } | |
| 401 | } | |
| 402 | ||
| 403 | // Download package icon | |
| 54d081 | 404 | //$icon = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='icon']/@href"); |
| TB | 405 | $icon = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='icon']/@href"); |
| 7fe908 | 406 | if($icon != '') |
| MC | 407 | { |
| 408 | $local_icon = $local_intf_folder.basename($icon); | |
| 409 | if(!file_exists($local_icon) || filesize($local_icon) == 0) | |
| 410 | { | |
| 411 | $apps_to_dl[] = array('name' => basename($icon), | |
| 412 | 'url' => $icon, | |
| 413 | 'filesize' => 0, | |
| 414 | 'localtarget' => $local_icon); | |
| 415 | } | |
| 416 | } | |
| 417 | ||
| 418 | // Download available screenshots | |
| 54d081 | 419 | //$screenshots = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='screenshot']", true); |
| TB | 420 | $screenshots = parent::getXPathValue($sxe, "entry[position()=" . $entry_pos . "]/link[@a:type='screenshot']", true); |
| 7fe908 | 421 | if(!empty($screenshots)) |
| MC | 422 | { |
| 423 | foreach($screenshots as $screen) | |
| 424 | { | |
| 425 | $local_screen = $local_intf_folder.basename($screen['href']); | |
| 426 | if(!file_exists($local_screen) || filesize($local_screen) == 0) | |
| 427 | { | |
| 428 | $apps_to_dl[] = array('name' => basename($screen['href']), | |
| 429 | 'url' => $screen['href'], | |
| 430 | 'filesize' => 0, | |
| 431 | 'localtarget' => $local_screen); | |
| 432 | } | |
| 433 | } | |
| 434 | } | |
| 435 | } | |
| 436 | else // Server mode (download whole ZIP archive) | |
| 437 | { | |
| 438 | // Delete an obviously out-dated version from the system | |
| 439 | if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) | |
| 440 | { | |
| 441 | $old_file = $this->packages_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; | |
| 442 | if(file_exists($old_file)) $this->removeDirectory($old_file); | |
| 443 | } | |
| 444 | ||
| 445 | // Attention: $new_ver can also be == $ex_ver (according to version_compare >= 0) | |
| 446 | $local_zip = $this->packages_dir.'/'.$app_name.'-'.$new_ver.'.app.zip'; | |
| 447 | ||
| 448 | // Before re-downloading a file, make sure it's not yet existing on HDD (due to DB inconsistency) | |
| 449 | if((file_exists($local_zip) && (filesize($local_zip) == $app_filesize)) === false) | |
| 450 | { | |
| 451 | $apps_to_dl[] = array('name' => $app_name, | |
| 452 | 'url' => $app_dl, | |
| 453 | 'filesize' => $app_filesize, | |
| 454 | 'localtarget' => $local_zip); | |
| 455 | $apps_downloaded++; | |
| 456 | } | |
| 457 | } | |
| 458 | } | |
| 459 | ||
| 460 | unset($sxe); | |
| 461 | $apps_in_repo++; | |
| 42f0c9 | 462 | } catch (Exception $e) { |
| TB | 463 | // We dont want the crawler to fail on xml parse errors |
| 464 | $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_WARN); | |
| 465 | //echo 'Caught exception: ', $e->getMessage(), "\n"; | |
| 466 | } | |
| 7fe908 | 467 | } |
| MC | 468 | } |
| 469 | //var_dump($apps); | |
| 42f0c9 | 470 | //echo print_r($apps_to_dl).'<br>-------------------<br>'; |
| 7fe908 | 471 | |
| MC | 472 | // For memory reasons, unset the current vendor and his apps |
| 473 | unset($apps); | |
| 474 | } | |
| 475 | ||
| 476 | // Shuffle the download array (in order to compensate unexpected php aborts) | |
| 477 | shuffle($apps_to_dl); | |
| 478 | ||
| 479 | // After collecting all provisioned apps, download them | |
| 480 | $apps_to_dl_chunks = array_chunk($apps_to_dl, 10); | |
| 481 | ||
| 482 | for($i = 0; $i < count($apps_to_dl_chunks); $i++) | |
| 483 | { | |
| 484 | $this->fetchFiles($apps_to_dl_chunks[$i]); | |
| 485 | ||
| 486 | // Check the integrity of all downloaded files | |
| 487 | // but exclude cases where no filesize is available (i.e. screenshot or metafile download) | |
| 488 | for($j = 0; $j < count($apps_to_dl_chunks[$i]); $j++) | |
| 489 | { | |
| 490 | if($apps_to_dl_chunks[$i][$j]['filesize'] != 0 && | |
| 491 | $apps_to_dl_chunks[$i][$j]['filesize'] != filesize($apps_to_dl_chunks[$i][$j]['localtarget'])) | |
| 492 | { | |
| 493 | $app->log($this->log_prefix.' The filesize of the package "'. | |
| 494 | $apps_to_dl_chunks[$i][$j]['name'].'" is wrong. Download failure?', LOGLEVEL_WARN); | |
| 495 | } | |
| 496 | } | |
| 497 | } | |
| 498 | ||
| 499 | $app->log($this->log_prefix.'Processed '.$apps_in_repo. | |
| 500 | ' apps from the repo. Downloaded '.$apps_updated. | |
| 501 | ' updates, '.$apps_downloaded.' new apps'); | |
| 502 | } | |
| 503 | ||
| 504 | catch(Exception $e) | |
| 505 | { | |
| 506 | $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR); | |
| 507 | return false; | |
| 508 | } | |
| 509 | } | |
| 510 | ||
| 511 | ||
| 512 | ||
| 513 | /** | |
| 514 | * Read in all possible packages from the interface packages folder and | |
| 515 | * check if they are not ASP.net code (as this can't be processed). | |
| 516 | * | |
| 517 | * Note: There's no need to check if the packages to register are newer | |
| 518 | * than those in the database because this already happended in startCrawler() | |
| 519 | */ | |
| 520 | public function parseFolderToDB() | |
| 521 | { | |
| 522 | global $app; | |
| 523 | ||
| 524 | try | |
| 525 | { | |
| 526 | // This method must be used in interface mode | |
| 527 | if(!$this->interface_mode) return false; | |
| 528 | ||
| 529 | $pkg_list = array(); | |
| 530 | ||
| 531 | // Read in every package having a correct filename | |
| 532 | $temp_handle = @dir($this->interface_pkg_dir); | |
| 533 | if(!$temp_handle) throw new Exception('The temp directory is not accessible'); | |
| 534 | while($folder = $temp_handle->read()) | |
| 535 | if(substr($folder, -8) == '.app.zip') $pkg_list[] = $folder; | |
| 536 | $temp_handle->close(); | |
| 537 | ||
| 538 | // If no packages are available -> exception (because at this point there should exist packages) | |
| 539 | if(empty($pkg_list)) throw new Exception('No packages to read in'); | |
| 540 | ||
| 541 | // Get registered packages and mark non-existant packages with an error code to omit the install | |
| 542 | $existing_packages = array(); | |
| 543 | $path_query = $app->db->queryAllRecords('SELECT path AS Path FROM aps_packages;'); | |
| 544 | foreach($path_query as $path) $existing_packages[] = $path['Path']; | |
| 545 | $diff = array_diff($existing_packages, $pkg_list); | |
| 546 | foreach($diff as $todelete) { | |
| 547 | /*$app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_ERROR_NOMETA."' | |
| 146783 | 548 | WHERE path = '".$app->db->quote($todelete)."';");*/ |
| MC | 549 | $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE path = '".$app->db->quote($todelete)."';"); |
| 550 | $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_ERROR_NOMETA, 'id', $tmp['id']); | |
| 551 | unset($tmp); | |
| 552 | } | |
| 7fe908 | 553 | |
| MC | 554 | // Register all new packages |
| 555 | $new_packages = array_diff($pkg_list, $existing_packages); | |
| 556 | foreach($new_packages as $pkg) | |
| 557 | { | |
| 558 | // Load in meta file if existing and register its namespaces | |
| 559 | $metafile = $this->interface_pkg_dir.'/'.$pkg.'/APP-META.xml'; | |
| 560 | if(!file_exists($metafile)) | |
| 561 | { | |
| 562 | $app->log($this->log_prefix.'Cannot read metadata from '.$pkg, LOGLEVEL_ERROR); | |
| 563 | continue; | |
| 564 | } | |
| 565 | ||
| 566 | $metadata = file_get_contents($metafile); | |
| 567 | $metadata = str_replace("xmlns=", "ns=", $metadata); | |
| 568 | $sxe = new SimpleXMLElement($metadata); | |
| 569 | $namespaces = $sxe->getDocNamespaces(true); | |
| 570 | foreach($namespaces as $ns => $url) $sxe->registerXPathNamespace($ns, $url); | |
| 571 | ||
| 572 | // Insert the new package | |
| 573 | $pkg_name = parent::getXPathValue($sxe, 'name'); | |
| 574 | $pkg_category = parent::getXPathValue($sxe, '//category'); | |
| 575 | $pkg_version = parent::getXPathValue($sxe, 'version'); | |
| 576 | $pkg_release = parent::getXPathValue($sxe, 'release'); | |
| 146783 | 577 | //$pkg_url = $this->app_download_url_list[$pkg]; |
| MC | 578 | $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$pkg.'/PKG_URL'); |
| 7fe908 | 579 | |
| 146783 | 580 | /* |
| 7fe908 | 581 | $app->db->query("INSERT INTO `aps_packages` |
| MC | 582 | (`path`, `name`, `category`, `version`, `release`, `package_status`) VALUES |
| 146783 | 583 | ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."', |
| MC | 584 | '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."', |
| 585 | ".$app->db->quote($pkg_release).", ".PACKAGE_ENABLED.");"); | |
| 586 | */ | |
| 587 | // Insert only if data is complete | |
| 588 | if($pkg != '' && $pkg_name != '' && $pkg_category != '' && $pkg_version != '' && $pkg_release != '' && $pkg_url){ | |
| 7fe908 | 589 | $insert_data = "(`path`, `name`, `category`, `version`, `release`, `package_url`, `package_status`) VALUES |
| 146783 | 590 | ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."', |
| MC | 591 | '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."', |
| 592 | ".$app->db->quote($pkg_release).", '".$app->db->quote($pkg_url)."', ".PACKAGE_ENABLED.");"; | |
| 7fe908 | 593 | |
| 146783 | 594 | $app->db->datalogInsert('aps_packages', $insert_data, 'id'); |
| MC | 595 | } else { |
| 596 | if(file_exists($this->interface_pkg_dir.'/'.$pkg)) $this->removeDirectory($this->interface_pkg_dir.'/'.$pkg); | |
| 597 | } | |
| 7fe908 | 598 | } |
| MC | 599 | } |
| 600 | ||
| 601 | catch(Exception $e) | |
| 602 | { | |
| 603 | $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR); | |
| 146783 | 604 | $app->error($e->getMessage()); |
| 7fe908 | 605 | return false; |
| MC | 606 | } |
| 607 | } | |
| 608 | ||
| 609 | ||
| 610 | ||
| 146783 | 611 | /** |
| 7fe908 | 612 | * Add missing package URLs to database |
| MC | 613 | */ |
| 614 | public function fixURLs() | |
| 615 | { | |
| 616 | global $app; | |
| 617 | ||
| 618 | try | |
| 619 | { | |
| 620 | // This method must be used in interface mode | |
| 621 | if(!$this->interface_mode) return false; | |
| 622 | ||
| 623 | $incomplete_pkgs = $app->db->queryAllRecords("SELECT * FROM aps_packages WHERE package_url = ''"); | |
| 146783 | 624 | if(is_array($incomplete_pkgs) && !empty($incomplete_pkgs)){ |
| MC | 625 | foreach($incomplete_pkgs as $incomplete_pkg){ |
| 626 | $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$incomplete_pkg['path'].'/PKG_URL'); | |
| 627 | if($pkg_url != ''){ | |
| 604c0c | 628 | $app->db->datalogUpdate('aps_packages', "package_url = '".$app->db->quote($pkg_url)."'", 'id', $incomplete_pkg['id']); |
| 146783 | 629 | } |
| MC | 630 | } |
| 631 | } | |
| 7fe908 | 632 | } |
| MC | 633 | |
| 634 | catch(Exception $e) | |
| 635 | { | |
| 636 | $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR); | |
| 146783 | 637 | $app->error($e->getMessage()); |
| 7fe908 | 638 | return false; |
| MC | 639 | } |
| 640 | } | |
| 641 | ||
| 146783 | 642 | } |
| 7fe908 | 643 | |
| MC | 644 | ?> |
