| commit | author | age | ||
| 146783 | 1 | <?php |
| MC | 2 | /* |
| 3 | Copyright (c) 2012, ISPConfig UG | |
| 4 | Contributors: web wack creations, http://www.web-wack.at | |
| 5 | All rights reserved. | |
| 6 | ||
| 7 | Redistribution and use in source and binary forms, with or without modification, | |
| 8 | are permitted provided that the following conditions are met: | |
| 9 | ||
| 10 | * Redistributions of source code must retain the above copyright notice, | |
| 11 | this list of conditions and the following disclaimer. | |
| 12 | * Redistributions in binary form must reproduce the above copyright notice, | |
| 13 | this list of conditions and the following disclaimer in the documentation | |
| 14 | and/or other materials provided with the distribution. | |
| 15 | * Neither the name of ISPConfig nor the names of its contributors | |
| 16 | may be used to endorse or promote products derived from this software without | |
| 17 | specific prior written permission. | |
| 18 | ||
| 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND | |
| 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | |
| 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | |
| 22 | IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, | |
| 23 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, | |
| 24 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY | |
| 26 | OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING | |
| 27 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, | |
| 28 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 29 | */ | |
| 7fe908 | 30 | require_once 'aps_base.inc.php'; |
| 146783 | 31 | |
| MC | 32 | @set_time_limit(0); |
| 33 | @ignore_user_abort(1); | |
| 34 | ||
| 35 | class ApsCrawler extends ApsBase | |
| 36 | { | |
| 37 | ||
| 7fe908 | 38 | //public $app_download_url_list = array(); |
| 146783 | 39 | |
| 7fe908 | 40 | /** |
| MC | 41 | * Constructor |
| 42 | * | |
| 43 | * @param $app the application instance (db handle + log method) | |
| 44 | * @param $interface_mode act in interface (true) or server mode (false) | |
| 45 | */ | |
| 146783 | 46 | |
| MC | 47 | |
| 7fe908 | 48 | public function __construct($app, $interface_mode = false) |
| MC | 49 | { |
| 50 | parent::__construct($app, 'APS crawler: ', $interface_mode); | |
| 51 | } | |
| 146783 | 52 | |
| MC | 53 | |
| 54 | ||
| 7fe908 | 55 | /** |
| MC | 56 | * Before the cron is executed, make sure all necessary options are set |
| 57 | * and all functions (i.e. cURL) are available | |
| 58 | */ | |
| 59 | private function checkRequirements() | |
| 60 | { | |
| 61 | global $app; | |
| 146783 | 62 | |
| 7fe908 | 63 | try |
| MC | 64 | { |
| 65 | // Check if allow_url_fopen is enabled | |
| 66 | if(!@ini_get('allow_url_fopen')) throw new Exception('allow_url_fopen is not enabled'); | |
| 67 | // Check if the cURL module is available | |
| 68 | if(!function_exists('curl_version')) throw new Exception('cURL is not available'); | |
| 69 | ||
| 70 | // Check if used folders are writable | |
| 71 | if($this->interface_mode) | |
| 72 | { | |
| 73 | if(!is_writable($this->interface_pkg_dir)) | |
| 74 | throw new Exception('the folder '.basename($this->interface_pkg_dir).' is not writable'); | |
| 75 | } | |
| 76 | else | |
| 77 | { | |
| 78 | if(!is_writable($this->packages_dir)) | |
| 79 | throw new Exception('the folder '.basename($this->packages_dir).' is not writable'); | |
| 80 | } | |
| 81 | ||
| 82 | return true; | |
| 83 | } | |
| 84 | ||
| 85 | catch(Exception $e) | |
| 86 | { | |
| 87 | $app->log($this->log_prefix.'Aborting execution because '.$e->getMessage(), LOGLEVEL_ERROR); | |
| 88 | return false; | |
| 89 | } | |
| 90 | } | |
| 91 | ||
| 92 | ||
| 93 | ||
| 94 | /** | |
| 95 | * Remove a directory recursively | |
| 96 | * In case of error be silent | |
| 97 | * | |
| 98 | * @param $dir the directory to remove | |
| 99 | */ | |
| 100 | private function removeDirectory($dir) | |
| 101 | { | |
| 102 | if(is_dir($dir)) | |
| 103 | { | |
| 104 | $files = scandir($dir); | |
| 105 | foreach($files as $file) | |
| 106 | { | |
| 107 | if($file != '.' && $file != '..') | |
| 108 | if(filetype($dir.'/'.$file) == 'dir') $this->removeDirectory($dir.'/'.$file); | |
| 109 | else @unlink($dir.'/'.$file); | |
| 110 | } | |
| 111 | reset($files); | |
| 112 | @rmdir($dir); | |
| 113 | } | |
| 114 | } | |
| 115 | ||
| 116 | ||
| 117 | /** | |
| 118 | * Fetch HTML data from one or more given URLs | |
| 119 | * If a string is given, a string is returned, if an array of URLs should | |
| 120 | * be fetched, the responses of the parallel queries are returned as array | |
| 121 | * | |
| 122 | * @param $input the string or array to fetch | |
| 123 | * @return $ret a query response string or array | |
| 124 | */ | |
| 125 | private function fetchPage($input) | |
| 126 | { | |
| 127 | $ret = array(); | |
| 128 | $url = array(); | |
| 129 | $conn = array(); | |
| 130 | ||
| 131 | // Make sure we are working with an array, further on | |
| 132 | if(!is_array($input)) $url[] = $input; | |
| 133 | else $url = $input; | |
| 134 | ||
| 135 | // Build the single cURL handles and add them to a multi handle | |
| 136 | $mh = curl_multi_init(); | |
| 137 | for($i = 0; $i < count($url); $i++) | |
| 138 | { | |
| 139 | $conn[$i] = curl_init('http://'.$this->fetch_url.$url[$i]); | |
| 140 | curl_setopt($conn[$i], CURLOPT_RETURNTRANSFER, true); | |
| 141 | curl_multi_add_handle($mh, $conn[$i]); | |
| 142 | } | |
| 143 | ||
| 144 | $active = 0; | |
| 145 | do curl_multi_exec($mh, $active); | |
| 146 | while($active > 0); | |
| 147 | ||
| 148 | // Get the response(s) | |
| 149 | for($i = 0; $i < count($url); $i++) | |
| 150 | { | |
| 151 | $ret[$i] = curl_multi_getcontent($conn[$i]); | |
| 152 | curl_multi_remove_handle($mh, $conn[$i]); | |
| 153 | curl_close($conn[$i]); | |
| 154 | } | |
| 155 | curl_multi_close($mh); | |
| 156 | ||
| 157 | if(count($url) == 1) $ret = $ret[0]; | |
| 158 | ||
| 159 | return $ret; | |
| 160 | } | |
| 161 | ||
| 162 | ||
| 163 | ||
| 164 | /** | |
| 165 | * Fetch binary data from a given array | |
| 166 | * The data is retrieved in binary mode and | |
| 167 | * then directly written to an output file | |
| 168 | * | |
| 169 | * @param $input a specially structed array | |
| 170 | * @see $this->startUpdate() | |
| 171 | */ | |
| 172 | private function fetchFiles($input) | |
| 173 | { | |
| 174 | $fh = array(); | |
| 175 | $url = array(); | |
| 176 | $conn = array(); | |
| 177 | ||
| 178 | // Build the single cURL handles and add them to a multi handle | |
| 179 | $mh = curl_multi_init(); | |
| 180 | ||
| 181 | // Process each app | |
| 182 | for($i = 0; $i < count($input); $i++) | |
| 183 | { | |
| 184 | $conn[$i] = curl_init($input[$i]['url']); | |
| 185 | $fh[$i] = fopen($input[$i]['localtarget'], 'wb'); | |
| 186 | ||
| 187 | curl_setopt($conn[$i], CURLOPT_BINARYTRANSFER, true); | |
| 188 | curl_setopt($conn[$i], CURLOPT_FILE, $fh[$i]); | |
| 189 | curl_setopt($conn[$i], CURLOPT_TIMEOUT, 0); | |
| 190 | curl_setopt($conn[$i], CURLOPT_FAILONERROR, 1); | |
| 191 | curl_setopt($conn[$i], CURLOPT_FOLLOWLOCATION, 1); | |
| 192 | ||
| 193 | curl_multi_add_handle($mh, $conn[$i]); | |
| 194 | } | |
| 195 | ||
| 196 | $active = 0; | |
| 197 | do curl_multi_exec($mh, $active); | |
| 198 | while($active > 0); | |
| 199 | ||
| 200 | // Close the handles | |
| 201 | for($i = 0; $i < count($input); $i++) | |
| 202 | { | |
| 203 | fclose($fh[$i]); | |
| 204 | curl_multi_remove_handle($mh, $conn[$i]); | |
| 205 | curl_close($conn[$i]); | |
| 206 | } | |
| 207 | curl_multi_close($mh); | |
| 208 | } | |
| 209 | ||
| 210 | ||
| 211 | ||
| 212 | /** | |
| 213 | * A method to build query URLs out of a list of vendors | |
| 214 | * | |
| 215 | */ | |
| 216 | private function formatVendorCallback($array_item) | |
| 217 | { | |
| 218 | $array_item = str_replace(' ', '%20', $array_item); | |
| 219 | $array_item = str_replace('http://', '', $array_item); | |
| 220 | $array_item = '/'.$this->aps_version.'.atom?vendor='.$array_item.'&pageSize=100'; | |
| 221 | return $array_item; | |
| 222 | } | |
| 223 | ||
| 224 | ||
| 225 | ||
| 226 | /** | |
| 227 | * The main method which performs the actual crawling | |
| 228 | */ | |
| 229 | public function startCrawler() | |
| 230 | { | |
| 231 | global $app; | |
| 232 | ||
| 233 | try | |
| 234 | { | |
| 235 | // Make sure the requirements are given so that this script can execute | |
| 236 | $req_ret = $this->checkRequirements(); | |
| 237 | if(!$req_ret) return false; | |
| 238 | ||
| 239 | // Execute the open task and first fetch all vendors (APS catalog API 1.1, p. 12) | |
| 240 | $app->log($this->log_prefix.'Fetching data from '.$this->fetch_url); | |
| 241 | ||
| 242 | $vendor_page = $this->fetchPage('/all-app/'); //$vendor_page = $this->fetchPage('/'.$this->aps_version.'/'); | |
| 243 | preg_match_all("/\<a href=\"(.+)\/\" class=\"vendor\"/i", $vendor_page, $matches); | |
| 244 | $vendors = array_map('urldecode', $matches[1]); | |
| 245 | if(!$vendors) throw new Exception('Unable to fetch vendors. Aborting'); | |
| 246 | ||
| 247 | // Format all vendors for further processing (i.e. typo3.org -> /1.atom?vendor=typo3.org&pageSize=100 | |
| 248 | //array_walk($vendors, array($this, 'formatVendorCallback')); | |
| 146783 | 249 | if(is_array($vendors)) { |
| MC | 250 | foreach($vendors as $key => $array_item) { |
| 251 | $vendors[$key] = $this->formatVendorCallback($array_item); | |
| 252 | } | |
| 253 | } | |
| 254 | ||
| 7fe908 | 255 | // Process all vendors in chunks of 50 entries |
| MC | 256 | $vendor_chunks = array_chunk($vendors, 50); |
| 257 | //var_dump($vendor_chunks); | |
| 258 | ||
| 259 | // Get all known apps from the database and the highest known version | |
| 260 | // Note: A dirty hack is used for numerical sorting of the VARCHAR field Version: +0 -> cast | |
| 261 | // A longer but typesafe way would be: ORDER BY CAST(REPLACE(Version, '.', '') AS UNSIGNED) DESC | |
| 262 | $existing_apps = $app->db->queryAllRecords("SELECT * FROM ( | |
| 263 | SELECT name AS Name, CONCAT(version, '-', CAST(`release` AS CHAR)) AS CurrentVersion | |
| 146783 | 264 | FROM aps_packages ORDER BY REPLACE(version, '.', '')+0 DESC, `release` DESC |
| MC | 265 | ) as Versions GROUP BY name"); |
| 7fe908 | 266 | //var_dump($existing_apps); |
| MC | 267 | |
| 268 | // Used for statistics later | |
| 269 | $apps_in_repo = 0; | |
| 270 | $apps_updated = 0; | |
| 271 | $apps_downloaded = 0; | |
| 272 | ||
| 273 | $apps_to_dl = array(); | |
| 274 | ||
| 275 | for($i = 0; $i < count($vendor_chunks); $i++) | |
| 276 | { | |
| 277 | // Fetch all apps for the current chunk of vendors | |
| 278 | $apps = $this->fetchPage($vendor_chunks[$i]); | |
| 279 | ||
| 280 | for($j = 0; $j < count($apps); $j++) | |
| 281 | { | |
| 282 | // Before parsing, make sure it's worth the work by checking if at least one app exists | |
| 283 | $apps_count = substr_count($apps[$j], '<opensearch:totalResults>0</opensearch:totalResults>'); | |
| 284 | if($apps_count == 0) // obviously this vendor provides one or more apps | |
| 285 | { | |
| 286 | // Rename namespaces and register them | |
| 287 | $xml = str_replace("xmlns=", "ns=", $apps[$j]); | |
| 288 | $sxe = new SimpleXMLElement($xml); | |
| 289 | $namespaces = $sxe->getDocNamespaces(true); | |
| 290 | foreach($namespaces as $ns => $url) $sxe->registerXPathNamespace($ns, $url); | |
| 291 | ||
| 292 | // Fetching values of interest | |
| 293 | $app_name = parent::getXPathValue($sxe, 'entry[position()=1]/a:name'); | |
| 294 | $app_version = parent::getXPathValue($sxe, 'entry[position()=1]/a:version'); | |
| 295 | $app_release = parent::getXPathValue($sxe, 'entry[position()=1]/a:release'); | |
| 296 | ||
| 297 | // Find out a (possibly) existing package version | |
| 298 | $ex_ver = ''; | |
| 146783 | 299 | /* |
| 7fe908 | 300 | array_walk($existing_apps, |
| 146783 | 301 | create_function('$v, $k, $ex_ver', 'if($v["Name"] == "'.$app_name.'") $ex_ver = $v["CurrentVersion"];'), &$ex_ver); |
| MC | 302 | */ |
| 303 | if(is_array($existing_apps)) { | |
| 304 | foreach($existing_apps as $k => $v) { | |
| 305 | if($v["Name"] == $app_name) $ex_ver = $v["CurrentVersion"]; | |
| 306 | } | |
| 307 | } | |
| 308 | ||
| 7fe908 | 309 | $new_ver = $app_version.'-'.$app_release; |
| MC | 310 | $local_intf_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$new_ver.'.app.zip/'; |
| 146783 | 311 | |
| 7fe908 | 312 | // Proceed if a newer or at least equal version has been found with server mode or |
| MC | 313 | // interface mode is activated and there are no valid APP-META.xml and PKG_URL existing yet |
| 314 | if((!$this->interface_mode && version_compare($new_ver, $ex_ver) >= 0) || ($this->interface_mode && (!file_exists($local_intf_folder.'APP-META.xml') || filesize($local_intf_folder.'APP-META.xml') == 0 || !file_exists($local_intf_folder.'PKG_URL') || filesize($local_intf_folder.'PKG_URL') == 0))){ | |
| 315 | // Check if we already have an old version of this app | |
| 316 | if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) $apps_updated++; | |
| 317 | ||
| 318 | $app_dl = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@href"); | |
| 319 | $app_filesize = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='aps']/@length"); | |
| 320 | $app_metafile = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='meta']/@href"); | |
| 321 | ||
| 146783 | 322 | //$this->app_download_url_list[$app_name.'-'.$new_ver.'.app.zip'] = $app_dl; |
| 7fe908 | 323 | // Skip ASP.net packages because they can't be used at all |
| MC | 324 | $asp_handler = parent::getXPathValue($sxe, '//aspnet:handler'); |
| 325 | $asp_permissions = parent::getXPathValue($sxe, '//aspnet:permissions'); | |
| 326 | $asp_version = parent::getXPathValue($sxe, '//aspnet:version'); | |
| 327 | if(!empty($asp_handler) || !empty($asp_permissions) || !empty($asp_version)) continue; | |
| 146783 | 328 | |
| 7fe908 | 329 | // Interface mode (download only parts) |
| MC | 330 | if($this->interface_mode) |
| 331 | { | |
| 332 | // Delete an obviously out-dated version from the system and DB | |
| 333 | if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) | |
| 334 | { | |
| 335 | $old_folder = $this->interface_pkg_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; | |
| 336 | if(file_exists($old_folder)) $this->removeDirectory($old_folder); | |
| 337 | ||
| 146783 | 338 | /* |
| MC | 339 | $app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_OUTDATED."' WHERE name = '". |
| 340 | $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". | |
| 341 | $app->db->quote($ex_ver)."';"); | |
| 342 | */ | |
| 343 | $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE name = '". | |
| 7fe908 | 344 | $app->db->quote($app_name)."' AND CONCAT(version, '-', CAST(`release` AS CHAR)) = '". |
| MC | 345 | $app->db->quote($ex_ver)."';"); |
| 146783 | 346 | $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_OUTDATED, 'id', $tmp['id']); |
| MC | 347 | unset($tmp); |
| 7fe908 | 348 | } |
| MC | 349 | |
| 350 | // Create the local folder if not yet existing | |
| 351 | if(!file_exists($local_intf_folder)) @mkdir($local_intf_folder, 0777, true); | |
| 352 | ||
| 146783 | 353 | // Save the package URL in an extra file because it's not part of the APP-META.xml file |
| MC | 354 | @file_put_contents($local_intf_folder.'PKG_URL', $app_dl); |
| 355 | ||
| 7fe908 | 356 | // Download the meta file |
| MC | 357 | $local_metafile = $local_intf_folder.'APP-META.xml'; |
| 358 | if(!file_exists($local_metafile) || filesize($local_metafile) == 0) | |
| 359 | { | |
| 360 | $apps_to_dl[] = array('name' => 'APP-META.xml', | |
| 361 | 'url' => $app_metafile, | |
| 362 | 'filesize' => 0, | |
| 363 | 'localtarget' => $local_metafile); | |
| 364 | $apps_downloaded++; | |
| 365 | } | |
| 366 | ||
| 367 | // Download package license | |
| 368 | $license = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='eula']/@href"); | |
| 369 | if($license != '') | |
| 370 | { | |
| 371 | $local_license = $local_intf_folder.'LICENSE'; | |
| 372 | if(!file_exists($local_license) || filesize($local_license) == 0) | |
| 373 | { | |
| 374 | $apps_to_dl[] = array('name' => basename($license), | |
| 375 | 'url' => $license, | |
| 376 | 'filesize' => 0, | |
| 377 | 'localtarget' => $local_license); | |
| 378 | } | |
| 379 | } | |
| 380 | ||
| 381 | // Download package icon | |
| 382 | $icon = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='icon']/@href"); | |
| 383 | if($icon != '') | |
| 384 | { | |
| 385 | $local_icon = $local_intf_folder.basename($icon); | |
| 386 | if(!file_exists($local_icon) || filesize($local_icon) == 0) | |
| 387 | { | |
| 388 | $apps_to_dl[] = array('name' => basename($icon), | |
| 389 | 'url' => $icon, | |
| 390 | 'filesize' => 0, | |
| 391 | 'localtarget' => $local_icon); | |
| 392 | } | |
| 393 | } | |
| 394 | ||
| 395 | // Download available screenshots | |
| 396 | $screenshots = parent::getXPathValue($sxe, "entry[position()=1]/link[@a:type='screenshot']", true); | |
| 397 | if(!empty($screenshots)) | |
| 398 | { | |
| 399 | foreach($screenshots as $screen) | |
| 400 | { | |
| 401 | $local_screen = $local_intf_folder.basename($screen['href']); | |
| 402 | if(!file_exists($local_screen) || filesize($local_screen) == 0) | |
| 403 | { | |
| 404 | $apps_to_dl[] = array('name' => basename($screen['href']), | |
| 405 | 'url' => $screen['href'], | |
| 406 | 'filesize' => 0, | |
| 407 | 'localtarget' => $local_screen); | |
| 408 | } | |
| 409 | } | |
| 410 | } | |
| 411 | } | |
| 412 | else // Server mode (download whole ZIP archive) | |
| 413 | { | |
| 414 | // Delete an obviously out-dated version from the system | |
| 415 | if(!empty($ex_ver) && version_compare($new_ver, $ex_ver) == 1) | |
| 416 | { | |
| 417 | $old_file = $this->packages_dir.'/'.$app_name.'-'.$ex_ver.'.app.zip'; | |
| 418 | if(file_exists($old_file)) $this->removeDirectory($old_file); | |
| 419 | } | |
| 420 | ||
| 421 | // Attention: $new_ver can also be == $ex_ver (according to version_compare >= 0) | |
| 422 | $local_zip = $this->packages_dir.'/'.$app_name.'-'.$new_ver.'.app.zip'; | |
| 423 | ||
| 424 | // Before re-downloading a file, make sure it's not yet existing on HDD (due to DB inconsistency) | |
| 425 | if((file_exists($local_zip) && (filesize($local_zip) == $app_filesize)) === false) | |
| 426 | { | |
| 427 | $apps_to_dl[] = array('name' => $app_name, | |
| 428 | 'url' => $app_dl, | |
| 429 | 'filesize' => $app_filesize, | |
| 430 | 'localtarget' => $local_zip); | |
| 431 | $apps_downloaded++; | |
| 432 | } | |
| 433 | } | |
| 434 | } | |
| 435 | ||
| 436 | unset($sxe); | |
| 437 | $apps_in_repo++; | |
| 438 | } | |
| 439 | } | |
| 440 | //var_dump($apps); | |
| 441 | ||
| 442 | // For memory reasons, unset the current vendor and his apps | |
| 443 | unset($apps); | |
| 444 | } | |
| 445 | ||
| 446 | // Shuffle the download array (in order to compensate unexpected php aborts) | |
| 447 | shuffle($apps_to_dl); | |
| 448 | ||
| 449 | // After collecting all provisioned apps, download them | |
| 450 | $apps_to_dl_chunks = array_chunk($apps_to_dl, 10); | |
| 451 | ||
| 452 | for($i = 0; $i < count($apps_to_dl_chunks); $i++) | |
| 453 | { | |
| 454 | $this->fetchFiles($apps_to_dl_chunks[$i]); | |
| 455 | ||
| 456 | // Check the integrity of all downloaded files | |
| 457 | // but exclude cases where no filesize is available (i.e. screenshot or metafile download) | |
| 458 | for($j = 0; $j < count($apps_to_dl_chunks[$i]); $j++) | |
| 459 | { | |
| 460 | if($apps_to_dl_chunks[$i][$j]['filesize'] != 0 && | |
| 461 | $apps_to_dl_chunks[$i][$j]['filesize'] != filesize($apps_to_dl_chunks[$i][$j]['localtarget'])) | |
| 462 | { | |
| 463 | $app->log($this->log_prefix.' The filesize of the package "'. | |
| 464 | $apps_to_dl_chunks[$i][$j]['name'].'" is wrong. Download failure?', LOGLEVEL_WARN); | |
| 465 | } | |
| 466 | } | |
| 467 | } | |
| 468 | ||
| 469 | $app->log($this->log_prefix.'Processed '.$apps_in_repo. | |
| 470 | ' apps from the repo. Downloaded '.$apps_updated. | |
| 471 | ' updates, '.$apps_downloaded.' new apps'); | |
| 472 | } | |
| 473 | ||
| 474 | catch(Exception $e) | |
| 475 | { | |
| 476 | $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR); | |
| 477 | return false; | |
| 478 | } | |
| 479 | } | |
| 480 | ||
| 481 | ||
| 482 | ||
| 483 | /** | |
| 484 | * Read in all possible packages from the interface packages folder and | |
| 485 | * check if they are not ASP.net code (as this can't be processed). | |
| 486 | * | |
| 487 | * Note: There's no need to check if the packages to register are newer | |
| 488 | * than those in the database because this already happended in startCrawler() | |
| 489 | */ | |
| 490 | public function parseFolderToDB() | |
| 491 | { | |
| 492 | global $app; | |
| 493 | ||
| 494 | try | |
| 495 | { | |
| 496 | // This method must be used in interface mode | |
| 497 | if(!$this->interface_mode) return false; | |
| 498 | ||
| 499 | $pkg_list = array(); | |
| 500 | ||
| 501 | // Read in every package having a correct filename | |
| 502 | $temp_handle = @dir($this->interface_pkg_dir); | |
| 503 | if(!$temp_handle) throw new Exception('The temp directory is not accessible'); | |
| 504 | while($folder = $temp_handle->read()) | |
| 505 | if(substr($folder, -8) == '.app.zip') $pkg_list[] = $folder; | |
| 506 | $temp_handle->close(); | |
| 507 | ||
| 508 | // If no packages are available -> exception (because at this point there should exist packages) | |
| 509 | if(empty($pkg_list)) throw new Exception('No packages to read in'); | |
| 510 | ||
| 511 | // Get registered packages and mark non-existant packages with an error code to omit the install | |
| 512 | $existing_packages = array(); | |
| 513 | $path_query = $app->db->queryAllRecords('SELECT path AS Path FROM aps_packages;'); | |
| 514 | foreach($path_query as $path) $existing_packages[] = $path['Path']; | |
| 515 | $diff = array_diff($existing_packages, $pkg_list); | |
| 516 | foreach($diff as $todelete) { | |
| 517 | /*$app->db->query("UPDATE aps_packages SET package_status = '".PACKAGE_ERROR_NOMETA."' | |
| 146783 | 518 | WHERE path = '".$app->db->quote($todelete)."';");*/ |
| MC | 519 | $tmp = $app->db->queryOneRecord("SELECT id FROM aps_packages WHERE path = '".$app->db->quote($todelete)."';"); |
| 520 | $app->db->datalogUpdate('aps_packages', "package_status = ".PACKAGE_ERROR_NOMETA, 'id', $tmp['id']); | |
| 521 | unset($tmp); | |
| 522 | } | |
| 7fe908 | 523 | |
| MC | 524 | // Register all new packages |
| 525 | $new_packages = array_diff($pkg_list, $existing_packages); | |
| 526 | foreach($new_packages as $pkg) | |
| 527 | { | |
| 528 | // Load in meta file if existing and register its namespaces | |
| 529 | $metafile = $this->interface_pkg_dir.'/'.$pkg.'/APP-META.xml'; | |
| 530 | if(!file_exists($metafile)) | |
| 531 | { | |
| 532 | $app->log($this->log_prefix.'Cannot read metadata from '.$pkg, LOGLEVEL_ERROR); | |
| 533 | continue; | |
| 534 | } | |
| 535 | ||
| 536 | $metadata = file_get_contents($metafile); | |
| 537 | $metadata = str_replace("xmlns=", "ns=", $metadata); | |
| 538 | $sxe = new SimpleXMLElement($metadata); | |
| 539 | $namespaces = $sxe->getDocNamespaces(true); | |
| 540 | foreach($namespaces as $ns => $url) $sxe->registerXPathNamespace($ns, $url); | |
| 541 | ||
| 542 | // Insert the new package | |
| 543 | $pkg_name = parent::getXPathValue($sxe, 'name'); | |
| 544 | $pkg_category = parent::getXPathValue($sxe, '//category'); | |
| 545 | $pkg_version = parent::getXPathValue($sxe, 'version'); | |
| 546 | $pkg_release = parent::getXPathValue($sxe, 'release'); | |
| 146783 | 547 | //$pkg_url = $this->app_download_url_list[$pkg]; |
| MC | 548 | $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$pkg.'/PKG_URL'); |
| 7fe908 | 549 | |
| 146783 | 550 | /* |
| 7fe908 | 551 | $app->db->query("INSERT INTO `aps_packages` |
| MC | 552 | (`path`, `name`, `category`, `version`, `release`, `package_status`) VALUES |
| 146783 | 553 | ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."', |
| MC | 554 | '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."', |
| 555 | ".$app->db->quote($pkg_release).", ".PACKAGE_ENABLED.");"); | |
| 556 | */ | |
| 557 | // Insert only if data is complete | |
| 558 | if($pkg != '' && $pkg_name != '' && $pkg_category != '' && $pkg_version != '' && $pkg_release != '' && $pkg_url){ | |
| 7fe908 | 559 | $insert_data = "(`path`, `name`, `category`, `version`, `release`, `package_url`, `package_status`) VALUES |
| 146783 | 560 | ('".$app->db->quote($pkg)."', '".$app->db->quote($pkg_name)."', |
| MC | 561 | '".$app->db->quote($pkg_category)."', '".$app->db->quote($pkg_version)."', |
| 562 | ".$app->db->quote($pkg_release).", '".$app->db->quote($pkg_url)."', ".PACKAGE_ENABLED.");"; | |
| 7fe908 | 563 | |
| 146783 | 564 | $app->db->datalogInsert('aps_packages', $insert_data, 'id'); |
| MC | 565 | } else { |
| 566 | if(file_exists($this->interface_pkg_dir.'/'.$pkg)) $this->removeDirectory($this->interface_pkg_dir.'/'.$pkg); | |
| 567 | } | |
| 7fe908 | 568 | } |
| MC | 569 | } |
| 570 | ||
| 571 | catch(Exception $e) | |
| 572 | { | |
| 573 | $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR); | |
| 146783 | 574 | $app->error($e->getMessage()); |
| 7fe908 | 575 | return false; |
| MC | 576 | } |
| 577 | } | |
| 578 | ||
| 579 | ||
| 580 | ||
| 146783 | 581 | /** |
| 7fe908 | 582 | * Add missing package URLs to database |
| MC | 583 | */ |
| 584 | public function fixURLs() | |
| 585 | { | |
| 586 | global $app; | |
| 587 | ||
| 588 | try | |
| 589 | { | |
| 590 | // This method must be used in interface mode | |
| 591 | if(!$this->interface_mode) return false; | |
| 592 | ||
| 593 | $incomplete_pkgs = $app->db->queryAllRecords("SELECT * FROM aps_packages WHERE package_url = ''"); | |
| 146783 | 594 | if(is_array($incomplete_pkgs) && !empty($incomplete_pkgs)){ |
| MC | 595 | foreach($incomplete_pkgs as $incomplete_pkg){ |
| 596 | $pkg_url = @file_get_contents($this->interface_pkg_dir.'/'.$incomplete_pkg['path'].'/PKG_URL'); | |
| 597 | if($pkg_url != ''){ | |
| 604c0c | 598 | $app->db->datalogUpdate('aps_packages', "package_url = '".$app->db->quote($pkg_url)."'", 'id', $incomplete_pkg['id']); |
| 146783 | 599 | } |
| MC | 600 | } |
| 601 | } | |
| 7fe908 | 602 | } |
| MC | 603 | |
| 604 | catch(Exception $e) | |
| 605 | { | |
| 606 | $app->log($this->log_prefix.$e->getMessage(), LOGLEVEL_ERROR); | |
| 146783 | 607 | $app->error($e->getMessage()); |
| 7fe908 | 608 | return false; |
| MC | 609 | } |
| 610 | } | |
| 611 | ||
| 146783 | 612 | } |
| 7fe908 | 613 | |
| MC | 614 | ?> |
