D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
home
/
mybf1
/
www
/
generator
/
pages
/
Filename :
class.grab.inc.php
back
Copy
<?php // This file is protected by copyright law and provided under license. Reverse engineering of this file is strictly prohibited. $wNyYxzmBvvcbLruLA=19974073;$rjwUHZn_1Qto=148655288;$ekGRVb4CT=581334285;$ISoA01oxRHNr3=573589709;$J98U5e8ssXMq=370509158;$GhrH0kGfcQ2fF557F9M=814853632;$QHUNcRO0gvbEVg=599023045;$bOiG9WaKwkLVIHo=511001948;$WWUwd8kZnR=885503198;$BkcGfxbUgIP3LZ=724199119;$CyYdHNynQJazhOcx=631652898;$zVDdiUD3wSri8EMipYU=888164452;$BOR_lm5n1aPr=74598005;$wMWUsifz2bA=180405502;$XTBaeL2_aKrDBPx=214986893;$mq2u39Y2IQbhpeW1=280838305;$xM8mFnxJMgEP=932261057;$X_zYm5SXlxdSYHTlvXA=673277597;$PO5ne_tQkS6=131185931;$vsl4NOIUZcaZIDpg=238626647;$DpLFqj2ejk5KACwhG=866158856;$y0RMrSsB6IPyke=711182522;$uQghLazi9c4n=674069980;$OnHcIkkAeJ=669801075;$aDYy70Vs493sv=34162952;$P49N_a_YSNVyetSGp=421843016;$egvpJiQ2VMdb=805918540;$N3UdQQtwATysl3ui=487727494;$oX1dXA3dbORGd0a=909241107;$eHeGZlwEM1lb5=290919348;$PIgkSg4lC4yyHppmxY=505093364;$a0fW1Ao4XMdv8N=602125720;$bK2P9t2ud00e=131814613;$uEK9jPSDaltaP=9866930;$kKxEnRtLR=641568788;$ip9bTZ_cv=407077779;$vBCzhBoSR1OMuu=460072213;$z7MDmwwKVK7H6WMRrP=784804110;$zrewLg_GM_2bd8=278937464;$GCXnfyME1k=79312592;$jCZ8qcKDNYeWmuuRZ=507354874;$iCAmoCZfufcvBQDkj=769910988;$wgU1Qgnfb9=66523504;$Fki2FY_6tm1zkjhDjA_=338303172;$jcdUZF0ohmm7me_QiOj=868082292;$sjtyolzHUPJQD=692667466;$yXxfpSWk5LG3w=990124122;$sHqX0fDxiwADp=527769072;$RNa6x2Hiqz=661395473;$T9FU4rV2npVuv=160980958;?><?php if(!defined('HqmBMPQB4QfPS'))return;class SiteCrawler { var $vY9pUGJSxmy8j = false; var $Ref0x8DrCYuXO = array(); var $X76u1YKBMGYbXmzi6sH = array(); var $SVH4BzIyh56a_HEGqt = array(); var $Te7nRdc2NfOAPzQBZFq = false; var $L2rCBT7py = false; var $Dku6hTsz64joltRugO = array(); var $x15LfqXVsH = array(); var $MVZvUfEx6bYBJwmz = ''; var $heFEaz03xhYW = ''; var $f5YhQAzju8oZNGeLb1K = ''; var $v0ytSBWER = ''; var $bpBWmF7wSzh7j_g = ''; var $aMX8JQFnFnMWTuOJ = ''; var $r7kkCXZmM = ''; var $kvnmYTY2GKJBcSP_1MK = ''; var $A78wE6jgmrny = ''; var $wOd8ZRUj4KujS = ''; var $lOXW9WDJYx6ROZX = 0; var $urls_completed = array(); var $sm_base = array(); var $y5E6XiZB3w2DW4XFY = 0; var $ctime = 0; var $pl = 0; var $mu = 0; var $num_processed = 0; var $num_links_current_batch = 0; var $num_urls_processed_in_current_batch = 0; var $tsize = 0; var $links_level = 0; var $E7PbBLC9yunfxjh = 0; var $nettime = 0; var $htTmB_dxmeudFU = 0; var $fetch_no = 0; var $s44JV5vkSKmyf7Lv5zD = 0; var $addedcnt = array(); var $sm_sessions = array(); var $Wz_cKqOXnzYS0EpP9h5 = array(); var $progpar = array(); var $z2lXQjiZDXfX = array(); var $runstate = array(); var $ref_links_list = array(); var $ref_links_tmp = array(); var $ref_links_tmp2 = array(); function KiQEfebckxJVM($yEo2C_lvbv7cBmFUYj = false, $MEqT8rghezZe = false) { global $O5KGcmOfedFfF, $grab_parameters, $K5qP9_ZsHx, $m8, $ZEYlLjKXE7C; $b31VIMGBiogCgOF=array_sum(explode(' ', microtime())); $this->ctime = $b31VIMGBiogCgOF - $this->s44JV5vkSKmyf7Lv5zD; $_ut = ($this->ctime - $O5KGcmOfedFfF > 5); if( $_ut || $MEqT8rghezZe || ( ( (($this->num_links_current_batch==$this->num_urls_processed_in_current_batch) || ($this->pl==0) || ($this->num_processed==1) || ($this->num_processed%$grab_parameters['xs_progupdate'])==0) || ($this->lOXW9WDJYx6ROZX>=$this->y5E6XiZB3w2DW4XFY) ) && ($this->num_processed != $this->Wz_cKqOXnzYS0EpP9h5['num_processed']) && !$yEo2C_lvbv7cBmFUYj ) ) { $this->Wz_cKqOXnzYS0EpP9h5['num_processed'] = $this->num_processed; $O5KGcmOfedFfF = $this->ctime; $this->mu = function_exists('memory_get_usage') ? memory_get_usage() : '-'; $K5qP9_ZsHx = max($K5qP9_ZsHx, $this->mu); if($this->mu>$m8+1000000){ $m8 = $this->mu; $cc = ' style="color:red"'; }else $cc=''; if(intval($this->mu)) $this->mu = number_format($this->mu/1024,1).' Kb'; wkwPIGj7HFydE9Mly("\n(<span".$cc.">memory".($cc?' up':'').": $this->mu</span>)<br>\n"); $this->progpar = array( $this->ctime, // 0. running time str_replace($this->wOd8ZRUj4KujS, '', $this->aMX8JQFnFnMWTuOJ), // 1. current URL $this->pl, // 2. urls left $this->num_processed, // 3. processed urls $this->tsize, // 4. bandwidth usage $this->links_level, // 5. depth level $this->mu, // 6. memory usage $this->lOXW9WDJYx6ROZX, // 7. added in sitemap $this->E7PbBLC9yunfxjh, // 8. in the queue $this->nettime, // 9. network time $this->htTmB_dxmeudFU, // 10. last net time $this->fetch_no, // 11. fetched urls $this->addedcnt, // 12. number of added images/videos/etc ); end($this->sm_sessions); $pS3BE0xOAS = key($this->sm_sessions); if(!$this->sm_sessions[$pS3BE0xOAS]['progress_start']){ $this->sm_sessions[$pS3BE0xOAS]['progress_start'] = $this->progpar; } $this->sm_sessions[$pS3BE0xOAS]['progress_end'] = $this->progpar; if($this->SVH4BzIyh56a_HEGqt['bgexec']){ if(((time()-$ZEYlLjKXE7C)>wS0f9iXRiSA('xs_state_interval',5)) || $MEqT8rghezZe) { if($this->fetch_no) { $ZEYlLjKXE7C = time(); $this->progpar[] = DfjNmPlYlDtVYeKoRsL(); if($yEo2C_lvbv7cBmFUYj){ $this->progpar[] = $yEo2C_lvbv7cBmFUYj; } Ndm7I4IRr(m093Bbc4Eg,Ie0qgN_fQ8HA($this->progpar)); } } } if($this->vY9pUGJSxmy8j && (!$this->X76u1YKBMGYbXmzi6sH['f'] || $_ut)) { call_user_func($this->vY9pUGJSxmy8j,$this->progpar); } } else { call_user_func($this->vY9pUGJSxmy8j,array('cmd'=>'ping', 'bg' => $this->SVH4BzIyh56a_HEGqt['bgexec'])); } return $this->progpar; } function zclLIX3r6Q9pMvIeZ88() { global $grab_parameters; if($grab_parameters['xs_prev_sm_base']){ Ndm7I4IRr('sm_base.db',nygAbN82hn($this->sm_base),pbAiQcHEGrGKkyqo4Q,true); } } function HD0JbCxpxAMgmmIJ() { global $grab_parameters; if($grab_parameters['xs_prev_sm_base']){ if($this->basecachemask) foreach($this->urls_completed as $b_pbe8WWqjqbWNds8xT){ $gEBIZ5cHL = $this->TTzcaknXMPA5kKTlsHt($b_pbe8WWqjqbWNds8xT); $Nu97QVlZVSDLOVglnd = $this->KVz31hE2O($gEBIZ5cHL['link']); if(preg_match('#('.$this->basecachemask.')#',$Nu97QVlZVSDLOVglnd)) if(!isset($this->sm_base[$Nu97QVlZVSDLOVglnd])) { $this->sm_base[$Nu97QVlZVSDLOVglnd] = $gEBIZ5cHL; } } } } function R5xg8ekoBHxW($d2a2uMuqO3Fh0aLFoJY) { global $grab_parameters; if($grab_parameters['xs_prev_sm_base']){ wkwPIGj7HFydE9Mly("\nRestoring sm_base\n"); if($gdpJ5qR1VAOh6 = @raSnfm1S9eiZTlT(pbAiQcHEGrGKkyqo4Q.'sm_base.db',true)){ wkwPIGj7HFydE9Mly("\nUnpacked sm_base size: ".(strlen($gdpJ5qR1VAOh6))."\n"); $gdpJ5qR1VAOh6 = Za80dklcf36($gdpJ5qR1VAOh6); } wkwPIGj7HFydE9Mly("\nFound URLs in sm_base: ".count($gdpJ5qR1VAOh6)."\n"); $this->sm_base = array(); if($gdpJ5qR1VAOh6) { if($this->basecachemask) foreach($gdpJ5qR1VAOh6 as $_k=>$_v){ $Nu97QVlZVSDLOVglnd = $this->KVz31hE2O($_k); $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($_k, $this->A78wE6jgmrny); if(!$LV3feH44NvQ) if(preg_match('#('.$this->basecachemask.')#',$Nu97QVlZVSDLOVglnd)) { $this->sm_base[$Nu97QVlZVSDLOVglnd] = $_v; } } } unset($gdpJ5qR1VAOh6); wkwPIGj7HFydE9Mly("\nFiltered after cache mask: ".count($this->sm_base)."\n"); if($d2a2uMuqO3Fh0aLFoJY){ } } } function CkkDVwCnm_1R($bm = false){ if($bm && isset($bm[1])&& $bm[1]&&($bm[1][0]!='.')){ $this->bBAymp71W4mgyv_2c($bm[1], $this->aMX8JQFnFnMWTuOJ); $this->kvnmYTY2GKJBcSP_1MK = GnOmDgAJZQXc9AEMaJ($bm[1]); } else{ $this->kvnmYTY2GKJBcSP_1MK = GnOmDgAJZQXc9AEMaJ(strstr($this->aMX8JQFnFnMWTuOJ,'://') ? $this->aMX8JQFnFnMWTuOJ : $this->A78wE6jgmrny . $this->aMX8JQFnFnMWTuOJ); } } function PA0Gyvv3UI($K71jAZIWNQat1l9V){ return strstr($K71jAZIWNQat1l9V,'text/html') || strstr($K71jAZIWNQat1l9V,'/xhtml'); } function cIK1ca8fG0pg0S($nm, $cn){ preg_match('#<input[^>]*name="'.$nm.'"[^>]*value="(.*?)"#is', $cn, $_inpm); return $_inpm[1]; } function pBCEldH3gSIzsjeU($cn, $dp){ global $grab_parameters,$k0huGF8gg4LQ1A; $KcmpZavBWiSGlc = array(); wkwPIGj7HFydE9Mly("\n*** Parse JS - ".$this->aMX8JQFnFnMWTuOJ.", ($dp)\n", 2); if(strstr($cn, '__VIEWSTATE')){ $JuXZXx085_JZG = rand(1E5,1E6); preg_match_all('#<form[^>]*action="([^>]*?)".*?</form>#is', $cn, $_vsm); foreach($_vsm[0] as $i=>$_vm) { if(!preg_match('#<input[^>]*?"__VIEWSTATE"#is', $_vm)) continue; $_action = $_vsm[1][$i]; $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($_action, $this->aMX8JQFnFnMWTuOJ); if($LV3feH44NvQ == 1) continue; $_fex2 = $this->k7_EQIP1qhQCfrZAa($_action); if($_fex2['f'])continue; $QMIkZdXBPXkL = array('__VIEWSTATE','__VIEWSTATEGENERATOR','__EVENTVALIDATION'); $g2fDJDz_Tawj = array(); foreach($QMIkZdXBPXkL as $p) $g2fDJDz_Tawj[$p] = $this->cIK1ca8fG0pg0S($p, $_vm); preg_match_all('#<input[^>]*type="hidden"[^>]*name="([^>]*?)"[^>]*value="([^>]*?)"#is', $_vm, $_vpar); foreach($_vpar[0] as $_vpi=>$_vpv) { if(!$g2fDJDz_Tawj[$_vpar[1][$_vpi]]) $g2fDJDz_Tawj[$_vpar[1][$_vpi]] = $_vpar[2][$_vpi]; } preg_match_all('#__doPostBack(\(.*?\))#is', $_vm, $_dpball); $g1hSZNkHzKVMV_ = array(); foreach($_dpball[1] as $_dpb) { $_dpb = stripslashes(str_replace(''',"'",$_dpb)); if(preg_match('#\'(.*?)\'(?:\s*\,\'(.*?)\')?#', $_dpb, $_a)){ if(!$this->NxfMP9qoH('xs_parse_js_only_act', $_a[1], true)) continue; if(!$this->NxfMP9qoH('xs_parse_js_only_act2', $_a[1].$_a[2], true)) continue; $TXTWLnQNZuIApqYH = $_action .' -> '. $_a[1]. '/'. $_a[2]; if($RZw22HLFtuRk7BzMD0 = wS0f9iXRiSA('xs_parse_js_allowdup',false)){ if(preg_match('#'.$RZw22HLFtuRk7BzMD0.'#',$_a[1].$_a[2])) $TXTWLnQNZuIApqYH = $this->aMX8JQFnFnMWTuOJ.'/'.$JuXZXx085_JZG.' -> '.$TXTWLnQNZuIApqYH; } if($this->z2lXQjiZDXfX[$TXTWLnQNZuIApqYH]++) continue; $_fex2 = $this->k7_EQIP1qhQCfrZAa($_a[1]); if(!$_fex2['f'] ) $g1hSZNkHzKVMV_[] = array($_a,$TXTWLnQNZuIApqYH); } } foreach($g1hSZNkHzKVMV_ as $jp){ $_a = $jp[0]; $TXTWLnQNZuIApqYH = $jp[1]; $SVH4BzIyh56a_HEGqt = $g2fDJDz_Tawj; $SVH4BzIyh56a_HEGqt['__EVENTTARGET'] = $_a[1]; $SVH4BzIyh56a_HEGqt['__EVENTARGUMENT'] = $_a[2]; $uFxTSS9XSb52KZo7YrD = $k0huGF8gg4LQ1A->fetch($_action, 0, false, false, http_build_query($SVH4BzIyh56a_HEGqt,'','&'), array('contenttype'=>'application/x-www-form-urlencoded')); $this->KiQEfebckxJVM(); wkwPIGj7HFydE9Mly("\n(js post $dp) $TXTWLnQNZuIApqYH\nlast - ".$uFxTSS9XSb52KZo7YrD['last_url'] ); if(($_action != $uFxTSS9XSb52KZo7YrD['last_url'])){ $KcmpZavBWiSGlc[] = $uFxTSS9XSb52KZo7YrD['last_url']; }else { $KcmpZavBWiSGlc = array_merge($KcmpZavBWiSGlc, $this->am05AsFKZIyD($uFxTSS9XSb52KZo7YrD['content'], $dp+1)); } wkwPIGj7HFydE9Mly("\n>> llist ".count($KcmpZavBWiSGlc).", ".$KcmpZavBWiSGlc[0]."\n"); } } } return $KcmpZavBWiSGlc; } function am05AsFKZIyD(&$cn, $dp = 0, $W7vOufXV1e = 0){ global $grab_parameters,$k0huGF8gg4LQ1A; if(!$W7vOufXV1e) $W7vOufXV1e= wS0f9iXRiSA('xs_parsehtml_maxdepth',5); wkwPIGj7HFydE9Mly("\n* Parse HTML - ".$this->aMX8JQFnFnMWTuOJ.", ($dp/$W7vOufXV1e) [".strlen($cn)."]\n", 2); if($dp > $W7vOufXV1e)return array(); if(preg_match_all('#<(?:div|span)\s[^>]*?(?:data-|\s)src\s*=\s*["\']([^>]*?)("|\')#is', $cn, $y99N3SqLl)) { foreach($y99N3SqLl[1] as $i=>$S6k9tLUMCTek9QF) { $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($S6k9tLUMCTek9QF, $this->aMX8JQFnFnMWTuOJ); if($LV3feH44NvQ == 1) continue; $_fex2 = $this->k7_EQIP1qhQCfrZAa($S6k9tLUMCTek9QF); if($_fex2['f'])continue; $uvFjgZyHKM = $k0huGF8gg4LQ1A->fetch($S6k9tLUMCTek9QF, 0, false, false); if($this->PA0Gyvv3UI($uvFjgZyHKM['headers']['content-type'])){ $vyDPTD0FlDWOb_G = $y99N3SqLl[0][$i]; $OTVikTqWUqpprJPRSH = strpos($cn, $vyDPTD0FlDWOb_G); if ($OTVikTqWUqpprJPRSH !== false) { $cn = substr_replace($cn, $uvFjgZyHKM['content'], $OTVikTqWUqpprJPRSH, strlen($vyDPTD0FlDWOb_G)); } } } } $NPQb1A9Fyg = $grab_parameters['xs_utf8_enc'] ? 'isu':'is'; $_t = 'a|area|go'; $_t .= '|link'; preg_match_all('#<(?:'.$_t.')(?:[^>]*?\s)href\s*=\s*(?:"([^"]*)|\'([^\']*)|([^\s\"\\\\>]+))[^>]*>#is'.$NPQb1A9Fyg, $cn, $am); preg_match_all('#<option(?:[^>]*?)?value\s*=\s*"(http[^"]*)#is'.$NPQb1A9Fyg, $cn, $XuU7UBqrL_os8B_GdjT); $_sc = '(?:i?frame)'; preg_match_all('#<'.$_sc.'\s[^>]*?src\s*=\s*["\']?(.*?)("|>|\')#is', $cn, $L2HmGWTj9H); preg_match_all('#<meta\s[^>]*http-equiv\s*=\s*"?refresh[^>]*URL\s*=\s*["\']?(.*?)("|>|\'[>\s])#'.$NPQb1A9Fyg, $cn, $T9v4StE5HjmB1); if($grab_parameters['xs_parse_swf']) preg_match_all('#<object[^>]*application/x-shockwave-flash[^>]*data\s*=\s*["\']([^"\'>]+).*?>#'.$NPQb1A9Fyg, $cn, $p0Z32taip_p); else $p0Z32taip_p = array(array(),array()); preg_match_all('#<a[^>]*?onclick\s*=\s*"[^"]*\.load\(\'([^\']*)#'.$NPQb1A9Fyg, $cn, $V2Te5xd_m_); preg_match_all('#"url"\:"(http[^"]*)#is'.$NPQb1A9Fyg, $cn, $Z2zkjGP9kFkUfy); $KcmpZavBWiSGlc = array(); $H4tQMPyGoPl90rESIQ = 'stylesheet|publisher'; if(isset($grab_parameters['xs_robotstxt']) && $grab_parameters['xs_robotstxt']) $H4tQMPyGoPl90rESIQ .= '|nofollow'; for($i=0;$i<count($am[1]);$i++) { if(!preg_match('#rel\s*=\s*["\']?\s*('.$H4tQMPyGoPl90rESIQ.')#i', $am[0][$i])) $KcmpZavBWiSGlc[] = $am[1][$i]; } $KcmpZavBWiSGlc = @array_merge( $KcmpZavBWiSGlc, $am[2],$am[3], $L2HmGWTj9H[1],$T9v4StE5HjmB1[1], $XuU7UBqrL_os8B_GdjT[1],$V2Te5xd_m_[1], $Z2zkjGP9kFkUfy[1], $p0Z32taip_p[1]); if($dp < $W7vOufXV1e) if($grab_parameters['xs_parse_js'] && $this->NxfMP9qoH('xs_parse_js_only', $this->aMX8JQFnFnMWTuOJ, true) ){ $BKGcgJpnBw4nrvc7A = $this->pBCEldH3gSIzsjeU($cn, $dp); $KcmpZavBWiSGlc = @array_merge($KcmpZavBWiSGlc, $BKGcgJpnBw4nrvc7A); } wkwPIGj7HFydE9Mly("\n* Parsed [".count($KcmpZavBWiSGlc)."] URLs\n", 2); return $KcmpZavBWiSGlc; } function Ptt9vi0YbID0fqFe($tipYoohv3rzRi9){ return preg_replace('#^(www|\w)\.#', '', $tipYoohv3rzRi9); } function drKZs8x1lb3X3n($VTg9vpFl_1b_1OEu) { if(preg_match('#^(.*?\://[^/]*)#', $VTg9vpFl_1b_1OEu, $exq38FmOAtD3Vrwa)) return strtolower($exq38FmOAtD3Vrwa[1]) . substr($VTg9vpFl_1b_1OEu, strlen($exq38FmOAtD3Vrwa[1])); else return $VTg9vpFl_1b_1OEu; } function KVz31hE2O($VTg9vpFl_1b_1OEu) { return preg_replace('#^.*?'.preg_quote($this->A78wE6jgmrny,'#').'#','',$VTg9vpFl_1b_1OEu); } function bBAymp71W4mgyv_2c(&$a, $AKn7hFzTSwEhWIx, $b381QxsWK = '') { global $grab_parameters; $r7kkCXZmM = $this->r7kkCXZmM; $kvnmYTY2GKJBcSP_1MK = $this->kvnmYTY2GKJBcSP_1MK; $a1 = $a; $a = str_replace( array('™','&','&','&',':',':','/', '/','.', '.'), array('%E2%84%A2', '&', '&', '&', ':', ':', '/', '/','.','.') ,$a); if(strstr($AKn7hFzTSwEhWIx,'://')) { $r7kkCXZmM = preg_replace('#(:\/\/.*?)\/.*$#', '$01', $kvnmYTY2GKJBcSP_1MK); } $aZgciCYKAafyx = parse_url($this->A78wE6jgmrny); if($aZgciCYKAafyx['scheme'] && substr($a, 0, 2) == '//') $a = $aZgciCYKAafyx['scheme'].':'.$a; $ghESvJ9ZSnQ = @parse_url($a); $_scheme = strtolower($ghESvJ9ZSnQ['scheme']); if($_scheme && ($_scheme!='http')&& ($_scheme!='https')) { $LV3feH44NvQ = 1; }else { $a = str_replace(':80/', '/', $a); $a = str_replace(':443/', '/', $a); if($a[0]=='?')$a = preg_replace('#^([^\?]*?)([^/\?]*?)(\?.*)?$#','$2',$AKn7hFzTSwEhWIx).$a; if($grab_parameters['xs_inc_ajax'] && strstr($a,'#!')){ $kvnmYTY2GKJBcSP_1MK = preg_replace('#\#.*$#', '', $kvnmYTY2GKJBcSP_1MK); if($a[0] != '/' && !strstr($a,':/')) $a = $kvnmYTY2GKJBcSP_1MK . preg_replace('#^([^\#]*?/)?([^/\#]*)?(\#.*)?$#', '$2', $AKn7hFzTSwEhWIx).$a; } if(preg_match('#^https?(:|&\#58;)#is',$a)){ if(preg_match('#://[^/]*$#is',$a)) $a .= '/'; $a = preg_replace('#(://[^/]*/)/#is','$01',$a); } else if($a&& $a[0]=='/')$a = $r7kkCXZmM.$a; else $a = $kvnmYTY2GKJBcSP_1MK.$a; if($a[0]=='/')$a = $r7kkCXZmM.$a; $a=str_replace('/./','/',$a); $a=preg_replace('#/\.$#','/',$a); if(substr($a,-2) == '..')$a.='/'; if(strstr($a,'../')){ preg_match('#(.*?:.*?//.*?)(/.*)$#',$a,$aa); do{ $ap = $aa[2]; $aa[2] = preg_replace('#/?[^/]*/\.\.#','',$ap,1); }while($aa[2]!=$ap); $a = $aa[1].$aa[2]; } $a = preg_replace('#/\./#','/',$a); $a = preg_replace('#([^&])\#'.($grab_parameters['xs_inc_ajax']?'([^\!]|$)':'').'.*$#','$01',$a); $a = preg_replace('#^(/)/+#','\\1',$a); $a = preg_replace('#[\r\n]+#s','',$a); if($grab_parameters['xs_cleanurls']) $a = @preg_replace($grab_parameters['xs_cleanurls'],'',$a); if($grab_parameters['xs_lowercase_urls']){ $a = strtolower($a); } if($grab_parameters['xs_custom_replace']){ global $dtemijOBd; if(!isset($dtemijOBd)){ $_ar = preg_split('#[\r\n]+#', trim($grab_parameters['xs_custom_replace'])); $dtemijOBd = array(); foreach($_ar as $v){ $me = explode(' ', $v); if($me[0]&& $me[1])$dtemijOBd['#'.$me[0].'#'] = $me[1]; } } if($dtemijOBd){ $a = @preg_replace(array_keys($dtemijOBd),array_values($dtemijOBd),$a); } } if($grab_parameters['xs_cleanpar']) { do { $Jh2yJ9lJIwOSTdVY7X = $a; $a = @preg_replace('#[\\?\\&]('.$grab_parameters['xs_cleanpar'].')=[a-z0-9\%\-\+\.\_\=\/\,\*]*$#i','',$a); $a = @preg_replace('#([\\?\\&])('.$grab_parameters['xs_cleanpar'].')=[a-z0-9\%\-\+\.\_\=\/\,\*]*&#i','$1',$a); }while($a != $Jh2yJ9lJIwOSTdVY7X); $a = @preg_replace('#\?\&?$#','',$a); } $LV3feH44NvQ = (strtolower(substr($a,0,strlen($this->A78wE6jgmrny)) ) != strtolower($this->A78wE6jgmrny)) ? 1 : 0; if(($LV3feH44NvQ==1) && $grab_parameters['xs_allow_subdomains']){ $ghESvJ9ZSnQ = @parse_url($a); if($ghESvJ9ZSnQ['host'] && preg_match($qq='#^(.*?\.)?'.preg_quote($this->Ptt9vi0YbID0fqFe($aZgciCYKAafyx['host']),'#').'$#', $ghESvJ9ZSnQ['host']) ){ $LV3feH44NvQ = 2; } } if(($LV3feH44NvQ==1) && $grab_parameters['xs_allow_extscheme']){ $_i2 = preg_replace('#^.*?://#','',$this->A78wE6jgmrny); $_a2 = preg_replace('#^.*?://#','',$a); if(!$LV3feH44NvQ = strtolower(substr($_a2,0,strlen($_i2)) ) != strtolower($_i2)) $LV3feH44NvQ = 2; } if(($LV3feH44NvQ==1) && $b381QxsWK) { $WioOPDRzFJfKDqi = $this->J4vPRMHzK($b381QxsWK); if($WioOPDRzFJfKDqi && preg_match('#('.$WioOPDRzFJfKDqi.')#', $a)) $LV3feH44NvQ = 2; } } wkwPIGj7HFydE9Mly("<br/>($a -- [$a1] -- (isext) $LV3feH44NvQ - (initdir) ".$this->A78wE6jgmrny." (iurl) $AKn7hFzTSwEhWIx - (top) $r7kkCXZmM - (ibase) $kvnmYTY2GKJBcSP_1MK - (tophosts) [".$this->Ptt9vi0YbID0fqFe($aZgciCYKAafyx['host']).", ".$ghESvJ9ZSnQ['host']."])<br>\n",3); return $LV3feH44NvQ; } function oGNbg9vqexWio($AKn7hFzTSwEhWIx){ return strstr($AKn7hFzTSwEhWIx,'://') ? $AKn7hFzTSwEhWIx : $this->A78wE6jgmrny . $AKn7hFzTSwEhWIx; } function NxfMP9qoH($BehATAY7B, $AKn7hFzTSwEhWIx, $PHA4ABEfvVsTMXt0 = true){ global $GWqc5wGFGMENLwZhisl,$grab_parameters; $GWOsRPYsWBxaRDI = $PHA4ABEfvVsTMXt0; if($grab_parameters[$BehATAY7B]){ if(!isset($GWqc5wGFGMENLwZhisl[$BehATAY7B])){ $GWqc5wGFGMENLwZhisl[$BehATAY7B] = $grab_parameters[$BehATAY7B]; if(!preg_match('#[\*\$]#',$GWqc5wGFGMENLwZhisl[$BehATAY7B])) $GWqc5wGFGMENLwZhisl[$BehATAY7B] = preg_quote($GWqc5wGFGMENLwZhisl[$BehATAY7B],'#'); $GWqc5wGFGMENLwZhisl[$BehATAY7B] = '#'.str_replace(' ', '|', $GWqc5wGFGMENLwZhisl[$BehATAY7B]).'#'; } $GWOsRPYsWBxaRDI = preg_match($GWqc5wGFGMENLwZhisl[$BehATAY7B],$AKn7hFzTSwEhWIx); } return $GWOsRPYsWBxaRDI; } function J4vPRMHzK($g1G64oq0ZZFb1KK){ if(!isset($this->Ref0x8DrCYuXO[$g1G64oq0ZZFb1KK])){ $this->Ref0x8DrCYuXO[$g1G64oq0ZZFb1KK] = trim($g1G64oq0ZZFb1KK) ? preg_replace( '#(^|\|)/#', '$01(?:^|/)', preg_replace("#\s*[\r\n]+\s*#",'|', (strstr($s=trim($g1G64oq0ZZFb1KK),'*')?$s:preg_quote($s,'#'))) ) : ''; } return $this->Ref0x8DrCYuXO[$g1G64oq0ZZFb1KK]; } function aZg3xMnbC_i($nuhscqzRdxDYVl, $AKn7hFzTSwEhWIx) { $f = false; if($nuhscqzRdxDYVl) foreach($nuhscqzRdxDYVl as $bm) { if($f = ($f || preg_match('#^('.$bm.')#', $AKn7hFzTSwEhWIx, $_imatch))) break; } return $f ? $_imatch[1] : 0; } function MgKnEUlaw6_XfzaD($AKn7hFzTSwEhWIx) { $bm = $this->aZg3xMnbC_i($this->f5YhQAzju8oZNGeLb1K, $AKn7hFzTSwEhWIx); if($bm) { $km4308YcYeHeJX0pW3 = $this->aZg3xMnbC_i($this->botmatch_allow, $AKn7hFzTSwEhWIx); return $km4308YcYeHeJX0pW3 ? false : $bm; } return false; } function k7_EQIP1qhQCfrZAa(&$AKn7hFzTSwEhWIx) { global $grab_parameters; $us = ''; if(isset($this->x15LfqXVsH[$AKn7hFzTSwEhWIx])) $AKn7hFzTSwEhWIx =$this->x15LfqXVsH[$AKn7hFzTSwEhWIx]; $f = $this->Te7nRdc2NfOAPzQBZFq && preg_match('#'.$grab_parameters['xs_exc_skip'].'#i',$AKn7hFzTSwEhWIx); if($f&&!$us)$us = 'excl-ext'; if($this->MVZvUfEx6bYBJwmz&&!$f)$f=$f||@preg_match('#('.$this->MVZvUfEx6bYBJwmz.')#',$AKn7hFzTSwEhWIx); if($f&&!$us)$us = 'excl-mask'; if($this->heFEaz03xhYW && $f && $grab_parameters['xs_incl_force']) $f = !preg_match('#('.$this->heFEaz03xhYW.')#',$AKn7hFzTSwEhWIx); $f = $f || $this->MgKnEUlaw6_XfzaD($this->v0ytSBWER . $AKn7hFzTSwEhWIx); if($f&&!$us)$us = 'excl-bot'; $f2 = false; $gEBIZ5cHL = false; if(!$f) { $f2 = $this->L2rCBT7py && preg_match('#'.$grab_parameters['xs_inc_skip'].'#i',$AKn7hFzTSwEhWIx); if($this->heFEaz03xhYW && !$f2) { $f2 = $f2||(preg_match('#('.$this->heFEaz03xhYW.')#',$AKn7hFzTSwEhWIx)); } if($grab_parameters['xs_parse_only'] && !$f2 && $AKn7hFzTSwEhWIx && ($AKn7hFzTSwEhWIx!='/')) { $f2 = $f2 || !$this->NxfMP9qoH('xs_parse_only', $AKn7hFzTSwEhWIx, true); } } $f3 = false; if($this->noincmask)$f3=@preg_match('#('.$this->noincmask.')#',$AKn7hFzTSwEhWIx); return array('f' => $f, 'f2' => $f2, 'f3' => $f3, 'uskip' => $us); } function puPBJwgZjs1JB2(&$AKn7hFzTSwEhWIx) { global $grab_parameters; $f = $this->NxfMP9qoH('xs_botapi_exclude', $AKn7hFzTSwEhWIx, false) || // matches exclude !$this->NxfMP9qoH('xs_botapi_parse_only', $AKn7hFzTSwEhWIx, true); // OR not matches "parse only" return $f; } function dtRHNsRCOCS7gtKcKk3($gEBIZ5cHL) { return ($gEBIZ5cHL); } function TTzcaknXMPA5kKTlsHt($gEBIZ5cHL) { return Za80dklcf36($gEBIZ5cHL); } function VAsjehLyY7SfsjZNoL2($Io29DwFf3WcSyMtAl, $AKn7hFzTSwEhWIx){ global $grab_parameters; $uYc3fIoW5Z = max(1, intval($grab_parameters['xs_maxref'])); if(!isset($this->ref_links_tmp[$Io29DwFf3WcSyMtAl]) ) $this->ref_links_tmp[$Io29DwFf3WcSyMtAl] = array(); if(count($this->ref_links_tmp[$Io29DwFf3WcSyMtAl]) < $uYc3fIoW5Z) $this->ref_links_tmp[$Io29DwFf3WcSyMtAl][] = $AKn7hFzTSwEhWIx; if( $grab_parameters['xs_ref_list_store'] && ($_rlmax = $grab_parameters['xs_ref_list_max']) ){ if(!isset($this->ref_links_list[$Io29DwFf3WcSyMtAl]) && count($this->ref_links_list)<$_rlmax){ $this->ref_links_list[$Io29DwFf3WcSyMtAl] = array(); } if(isset($this->ref_links_list[$Io29DwFf3WcSyMtAl])) { $this->ref_links_list[$Io29DwFf3WcSyMtAl][-1]++; if(count($this->ref_links_list[$Io29DwFf3WcSyMtAl]) <= $uYc3fIoW5Z) { if(!in_array($this->aMX8JQFnFnMWTuOJ, $this->ref_links_list[$Io29DwFf3WcSyMtAl])) $this->ref_links_list[$Io29DwFf3WcSyMtAl][] = $AKn7hFzTSwEhWIx; } } } } function V_tRhpx5sI9puf7($SVH4BzIyh56a_HEGqt) { global $grab_parameters,$k0huGF8gg4LQ1A,$K5qP9_ZsHx; error_reporting(E_ALL&~E_NOTICE); $this->SVH4BzIyh56a_HEGqt = $SVH4BzIyh56a_HEGqt; @set_time_limit($grab_parameters['xs_exec_time']); if($this->SVH4BzIyh56a_HEGqt['bgexec']) { ignore_user_abort(true); } register_shutdown_function('xvQcTIF8B113J'); if(function_exists('ini_set')) { @ini_set("zlib.output_compression", 0); @ini_set("output_buffering", 0); } $this->KiQEfebckxJVM(0,true); $this->s44JV5vkSKmyf7Lv5zD = array_sum(explode(' ', microtime())); $starttime = $dgHgts6lFbG = time(); $K5qP9_ZsHx = $this->nettime = 0; $this->wOd8ZRUj4KujS = $this->SVH4BzIyh56a_HEGqt['initurl']; $this->y5E6XiZB3w2DW4XFY = $this->SVH4BzIyh56a_HEGqt['maxpg']>0 ? $this->SVH4BzIyh56a_HEGqt['maxpg'] : 1E10; $O5DEYzmlsLOVgGYR1jS = $this->SVH4BzIyh56a_HEGqt['maxdepth'] ? $this->SVH4BzIyh56a_HEGqt['maxdepth'] : -1; $this->vY9pUGJSxmy8j = $this->SVH4BzIyh56a_HEGqt['progress_callback']; $this->MVZvUfEx6bYBJwmz = $this->J4vPRMHzK($grab_parameters['xs_excl_urls']); $this->heFEaz03xhYW = $this->J4vPRMHzK($grab_parameters['xs_incl_urls']); $this->noincmask = $this->J4vPRMHzK($grab_parameters['xs_noincl_urls']); $this->baseincmask = $this->J4vPRMHzK($grab_parameters['xs_prev_sm_incl']); $this->basecachemask = $this->J4vPRMHzK($grab_parameters['xs_prev_sm_cache']); $Rcj3xnKl_qu = $A6DPJo_TlvfZL8j = array(); $Xmfk5aHUVgA7WxET = ''; $AXVeGxVDLaOnReA = preg_split('#[\r\n]+#', $grab_parameters['xs_ind_attr']); $this->allowcode = '#200'.($grab_parameters['xs_allow_httpcode']?'|'.$grab_parameters['xs_allow_httpcode']:'').'#'; $this->badreqcode = '#400|429'.($grab_parameters['xs_badreq_httpcode']?'|'.$grab_parameters['xs_badreq_httpcode']:'').'#'; $this->interruptcode = (($_tmp=$grab_parameters['xs_interrupt_httpcode'])?'#'.$_tmp.'#':''); if($grab_parameters['xs_memsave']) { if(!file_exists(GFrAnuUwfGw3mh)) mkdir(GFrAnuUwfGw3mh, 0777); else if($this->SVH4BzIyh56a_HEGqt['resume']=='') zY4U4qfszx17aI(GFrAnuUwfGw3mh, '.txt'); } foreach($AXVeGxVDLaOnReA as $ia) if($ia) { $is = explode(',', $ia); if($is[0][0]=='$') $xDgZ5hLZdrtqCTh = substr($is[0], 1); else $xDgZ5hLZdrtqCTh = str_replace(array('\\^', '\\$'), array('^','$'), preg_quote($is[0],'#')); $A6DPJo_TlvfZL8j[] = $xDgZ5hLZdrtqCTh; $Rcj3xnKl_qu[] = array('lm' => $is[1], 'f' => $is[2], 'p' => $is[3]); } if($A6DPJo_TlvfZL8j) $Xmfk5aHUVgA7WxET = '('.implode(')|(',$A6DPJo_TlvfZL8j).')'; $i1zq0dY3F9_zWWq = parse_url($this->wOd8ZRUj4KujS); if(!$i1zq0dY3F9_zWWq['path']){$this->wOd8ZRUj4KujS.='/';$i1zq0dY3F9_zWWq = parse_url($this->wOd8ZRUj4KujS);} $lzNE5TjQXAahK7H35aV = 0; do { $uFxTSS9XSb52KZo7YrD = $k0huGF8gg4LQ1A->fetch($this->wOd8ZRUj4KujS, 0, true, false, '', array('getinfo'=>true));// the first request is to skip session id $hj01UnPDZ2UcHRo4eLE = !preg_match($this->allowcode,$uFxTSS9XSb52KZo7YrD['code']); }while($hj01UnPDZ2UcHRo4eLE && ($lzNE5TjQXAahK7H35aV++ <2)); $this->runstate['info'] = $uFxTSS9XSb52KZo7YrD['info']; if($hj01UnPDZ2UcHRo4eLE) { $hj01UnPDZ2UcHRo4eLE = ''; foreach($uFxTSS9XSb52KZo7YrD['headers'] as $k=>$v) $hj01UnPDZ2UcHRo4eLE .= $k.': '.$v.'<br />'; return array( 'errmsg'=>'<b>There was an error while retrieving the URL specified:</b> '.$this->wOd8ZRUj4KujS.''. ($uFxTSS9XSb52KZo7YrD['errormsg']?'<br><b>Error message:</b> '.$uFxTSS9XSb52KZo7YrD['errormsg']:''). '<br><b>HTTP Code:</b><br>'.$uFxTSS9XSb52KZo7YrD['protoline']. '<br><b>HTTP headers:</b><br>'.$hj01UnPDZ2UcHRo4eLE. '<br><b>HTTP output:</b><br>'.$uFxTSS9XSb52KZo7YrD['content'] , ); } $this->bBAymp71W4mgyv_2c($uFxTSS9XSb52KZo7YrD['last_url'], $this->wOd8ZRUj4KujS); $this->wOd8ZRUj4KujS = $uFxTSS9XSb52KZo7YrD['last_url']; $this->urls_completed = array(); $urls_ext = array(); $this->urls_404 = array(); $this->r7kkCXZmM = $i1zq0dY3F9_zWWq['scheme'].'://'.$i1zq0dY3F9_zWWq['host'].((!$i1zq0dY3F9_zWWq['port'] || ($i1zq0dY3F9_zWWq['port']=='80'))?'':(':'.$i1zq0dY3F9_zWWq['port'])); $this->num_processed = $this->tsize = $retrno = $m8HTpd_0s = $N0PX75BqtEcrjK = $this->fetch_no = 0; $this->A78wE6jgmrny = XGuLWSESslGQjVANPx1($this->r7kkCXZmM.'/', GnOmDgAJZQXc9AEMaJ($i1zq0dY3F9_zWWq['path'])); $nqlGKC7b_ZAN7dg = parse_url($this->A78wE6jgmrny); $this->v0ytSBWER = preg_replace('#^.+://[^/]+#', '', $this->A78wE6jgmrny); $this->Dku6hTsz64joltRugO = $k0huGF8gg4LQ1A->fetch($this->wOd8ZRUj4KujS,0,true,true); $e3gVo4pzCVZy188p = str_replace($this->A78wE6jgmrny,'',$this->wOd8ZRUj4KujS); $urls_list_full = array($e3gVo4pzCVZy188p=>1); if(!$e3gVo4pzCVZy188p)$e3gVo4pzCVZy188p=''; $urls_list = array($e3gVo4pzCVZy188p=>1); $urls_list2 = $urls_list_skipped = array(); $this->x15LfqXVsH = array(); $this->links_level = 0; $MqhSd09MtOq4uT = $this->ref_links_tmp = $this->ref_links_tmp2 = $this->ref_links_list = array(); $ph9tZXA4lu = 0; $ELiT8ew5NjA = $this->y5E6XiZB3w2DW4XFY; if(!$grab_parameters['xs_progupdate'])$grab_parameters['xs_progupdate'] = 20; $this->f5YhQAzju8oZNGeLb1K = array(); $this->botmatch_allow = array(); if(isset($grab_parameters['xs_robotstxt']) && $grab_parameters['xs_robotstxt']) { $h1y09q4tisyr5MRcb = $k0huGF8gg4LQ1A->fetch($this->r7kkCXZmM.'/robots.txt'); if($this->r7kkCXZmM.'/' != $this->A78wE6jgmrny) { } $ra=preg_split('#user-agent:\s*#im',$h1y09q4tisyr5MRcb['content']); $eLrPzfTPKO97oQ0mH_W=$Ye5bqIxOG2ePHa=array(); $WYC4RmasUB1 = false; for($i=1;$i<count($ra);$i++){ preg_match('#^(\S+)(.*)$#s',$ra[$i],$mKnBJSwVaktZgvkEa); $fJXmitMs7XYPqiE = preg_match('#^googlebot$#is',$mKnBJSwVaktZgvkEa[1]); $bMR1qZsUSS5SL = ($mKnBJSwVaktZgvkEa[1]=='*'); if(!$bMR1qZsUSS5SL&&!$fJXmitMs7XYPqiE) continue; if($bMR1qZsUSS5SL && $WYC4RmasUB1) continue; if($fJXmitMs7XYPqiE){ $WYC4RmasUB1 = true; $eLrPzfTPKO97oQ0mH_W = $Ye5bqIxOG2ePHa = array(); } preg_match_all('#^disallow:\s*([^\#\r\n]*)#im',$mKnBJSwVaktZgvkEa[2],$rm); for($pi=0;$pi<count($rm[1]);$pi++) if($rm[1][$pi]) $eLrPzfTPKO97oQ0mH_W[] = str_replace('\\$','$', str_replace(' ','', str_replace('\\*','.*', preg_quote($rm[1][$pi],'#') ))); preg_match_all('#^allow:\s*(\S*)#im',$mKnBJSwVaktZgvkEa[2],$rm); for($pi=0;$pi<count($rm[1]);$pi++) if($rm[1][$pi] && ($rm[1][$pi]!='/')) { $Ye5bqIxOG2ePHa[] = str_replace('\\$','$', str_replace('\\*','.*', str_replace(' ','', preg_quote($rm[1][$pi],'#') ))); } } for($i=0;$i<count($eLrPzfTPKO97oQ0mH_W);$i+=200) $this->f5YhQAzju8oZNGeLb1K[]=implode('|', array_slice($eLrPzfTPKO97oQ0mH_W, $i,200)); for($i=0;$i<count($Ye5bqIxOG2ePHa);$i+=200) $this->botmatch_allow[]=implode('|', array_slice($Ye5bqIxOG2ePHa, $i,200)); } if($grab_parameters['xs_inc_ajax']) $grab_parameters['xs_proto_skip'] = str_replace( '\#', '\#(?:[^\!]|$)', $grab_parameters['xs_proto_skip']); $this->Te7nRdc2NfOAPzQBZFq = $grab_parameters['xs_exc_skip']!='\\.()'; $this->L2rCBT7py = $grab_parameters['xs_inc_skip']!='\\.()'; $grab_parameters['xs_inc_skip'] .= '(?:$|\?)'; $grab_parameters['xs_exc_skip'] .= '(?:$|\?)'; if($grab_parameters['xs_debug']) { $_GET['ddbg']=1; Z4wwdX_eBH(); } $epcQYfCl1 = 0; $this->CkkDVwCnm_1R(); $this->num_urls_processed_in_current_batch = 0; $this->num_links_current_batch = 1; Ndm7I4IRr(guXHCq5aVeE,''); $d2a2uMuqO3Fh0aLFoJY = false; if($this->SVH4BzIyh56a_HEGqt['resume']!=''){ wkwPIGj7HFydE9Mly("\nRestoring session\n"); $mFUcZxJGA4itwRbJ = @U2Jtr5yOrK(raSnfm1S9eiZTlT(pbAiQcHEGrGKkyqo4Q . TYR4q027D0OWvh, true)); if($mFUcZxJGA4itwRbJ) { $d2a2uMuqO3Fh0aLFoJY = true; echo 'Resuming the last session (last updated: '.date('Y-m-d H:i:s',$mFUcZxJGA4itwRbJ['time']).')'."\n"; unset($mFUcZxJGA4itwRbJ['sm_base']); extract($mFUcZxJGA4itwRbJ); foreach($mFUcZxJGA4itwRbJ as $k=>$v){ if(isset($this->$k)){ $this->$k=$v; unset($$v); } } $this->s44JV5vkSKmyf7Lv5zD-=$this->ctime; $epcQYfCl1 = $this->ctime; unset($mFUcZxJGA4itwRbJ); } } $iCGf7dEtSbU = 0; $this->R5xg8ekoBHxW($d2a2uMuqO3Fh0aLFoJY); if($_murls = $grab_parameters['xs_moreurls_test']){ $this->mu = preg_split('#[\r\n]+#', $_murls); foreach($this->mu as $YbDvNPDLdp){ $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($YbDvNPDLdp, $this->A78wE6jgmrny); if($LV3feH44NvQ!=1) { $YbDvNPDLdp = $this->KVz31hE2O($YbDvNPDLdp); $urls_list[$YbDvNPDLdp]++; } } } if($d2a2uMuqO3Fh0aLFoJY){ }else { if($_mu_s = $grab_parameters['xs_moreurls']) { $this->mu = preg_split('#[\r\n]+#', $_mu_s); foreach($this->mu as $YbDvNPDLdp){ $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($YbDvNPDLdp, $this->A78wE6jgmrny); if($LV3feH44NvQ!=1) { $YbDvNPDLdp = $this->KVz31hE2O($YbDvNPDLdp); $urls_list[$YbDvNPDLdp]++; } } } if($_mu_s = $grab_parameters['xs_moreurls_import']) { $this->mu_imp = preg_split('#[\r\n]+#', $_mu_s); foreach($this->mu_imp as $XjY0_j8Woov_L6wt9Z){ $_mdef = explode(';', $XjY0_j8Woov_L6wt9Z, 2); $sTZJRrBTadswKbwX6ii = array(); $EiKVQkP7zxHmIwYyV = $k0huGF8gg4LQ1A->fetch($_mdef[1]); switch($_mdef[0]){ case 'txt': $sTZJRrBTadswKbwX6ii = preg_split('#[\r\n]+#', $EiKVQkP7zxHmIwYyV['content']); break; } foreach($sTZJRrBTadswKbwX6ii as $YbDvNPDLdp) { $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($YbDvNPDLdp, $this->A78wE6jgmrny); if($LV3feH44NvQ!=1) { $YbDvNPDLdp = $this->KVz31hE2O($YbDvNPDLdp); $urls_list[$YbDvNPDLdp]++; } } } } if($grab_parameters['xs_prev_sm_base']){ $A44mMfpwrmjIAlOnHKv = count($urls_list); if(is_array($this->sm_base) && ($grab_parameters['xs_prev_sm_base_min']<count($this->sm_base)) ){ if($this->baseincmask) foreach($this->sm_base as $_u=>$_e){ if(preg_match('#('.$this->baseincmask.')#',$_u)) $urls_list[$_u]++; } } else $this->sm_base = array(); wkwPIGj7HFydE9Mly("\nAdded in urls list: ".(count($urls_list)-$A44mMfpwrmjIAlOnHKv)."\n"); } $iCGf7dEtSbU = count($urls_list); $urls_list_full = $urls_list; $this->num_links_current_batch = count($urls_list); } $Ar3WLiESdY = (wS0f9iXRiSA('xs_use_botapi', false) && wS0f9iXRiSA('xs_botapi_url', false)); $m7sJxx1ev = explode('|', $grab_parameters['xs_force_inc']); $SMLYdzM8Msiva534J4 = $utIVJaPZH = array(); $this->lOXW9WDJYx6ROZX = count($this->urls_completed); $this->E7PbBLC9yunfxjh = count($urls_list2); sleep(1); @cMo3XWEqPThU4Id(pbAiQcHEGrGKkyqo4Q.guXHCq5aVeE); $this->sm_sessions[] = array( 'start' => time(), 'instance' => $grab_parameters['xs_instance_id'] ); $this->KiQEfebckxJVM(0,true); if($urls_list) do { RYtkTQfZ1my('pre',true); RYtkTQfZ1my('pre1'); if($SMLYdzM8Msiva534J4) { $_ul = array_shift($SMLYdzM8Msiva534J4); list($this->aMX8JQFnFnMWTuOJ, $D2aBvrGfxJa6J32gUZJ) = $_ul; }else { $this->aMX8JQFnFnMWTuOJ = key($urls_list);$D2aBvrGfxJa6J32gUZJ=current($urls_list);next($urls_list); } $zN7QvVaBZeQp5ZF = ($D2aBvrGfxJa6J32gUZJ>0 && $D2aBvrGfxJa6J32gUZJ<1) ? $D2aBvrGfxJa6J32gUZJ : 0; $this->num_urls_processed_in_current_batch++; wkwPIGj7HFydE9Mly("\n[ ".$this->num_urls_processed_in_current_batch." - $this->aMX8JQFnFnMWTuOJ, $D2aBvrGfxJa6J32gUZJ] \n"); unset($urls_list[$this->aMX8JQFnFnMWTuOJ]); $oO9pt9QoVu0fjOUEU6 = VNVyRwmOg2q_22W($this->aMX8JQFnFnMWTuOJ); $wyhMGdq7QGd6 = false; $OgSj2zxK4Qo5Cc = ''; RYtkTQfZ1my('pre1',true); RYtkTQfZ1my('pre2a'); $uFxTSS9XSb52KZo7YrD = array(); $cn = ''; $this->X76u1YKBMGYbXmzi6sH = $_fex = $this->k7_EQIP1qhQCfrZAa($this->aMX8JQFnFnMWTuOJ); extract($_fex); if($f) $OgSj2zxK4Qo5Cc = $_fex['uskip']; RYtkTQfZ1my('pre2a',true); RYtkTQfZ1my('pre2b'); if(!$f && ($this->lOXW9WDJYx6ROZX>0) && ($gEBIZ5cHL = $this->sm_base[$this->aMX8JQFnFnMWTuOJ])){ $f2 = true; } if($this->Dku6hTsz64joltRugO && strstr($this->Dku6hTsz64joltRugO['content'],'header'))$f2 = true; RYtkTQfZ1my('pre2b',true); do { // zero-while, while(false) to allow break only $p0o_d8t8MHu9ndEDl = count($urls_list) + $this->E7PbBLC9yunfxjh; $YY6ZpVykum = $p0o_d8t8MHu9ndEDl + $this->lOXW9WDJYx6ROZX; if(!$f && !$f2) { $Q2yukaFyeBRHXGLoZz = ($this->y5E6XiZB3w2DW4XFY>=200000); $Xg99Dr7wV27xjesu = ($this->y5E6XiZB3w2DW4XFY<=1000); if($Xg99Dr7wV27xjesu && !$GLOBALS['forcepar_reset']){ $GLOBALS['forcepar_reset'] = true; if($m7sJxx1ev[1]) $m7sJxx1ev[1] *= 10; if($m7sJxx1ev[2]) $m7sJxx1ev[2] *= 10; } $rzR9rfqnQiRlGki = ($m7sJxx1ev[1] && (($this->ctime>$m7sJxx1ev[0]) && ($this->num_processed>$this->y5E6XiZB3w2DW4XFY*$m7sJxx1ev[1]))); $v7cZje3jo = $m7sJxx1ev[2] && ( $Q2yukaFyeBRHXGLoZz && // large sites only ($ELiT8ew5NjA*$m7sJxx1ev[2]+1000)< ($p0o_d8t8MHu9ndEDl - $this->num_urls_processed_in_current_batch));//-$iCGf7dEtSbU)); wkwPIGj7HFydE9Mly("force: (1,$rzR9rfqnQiRlGki)(2,$v7cZje3jo) pleft: ".$ELiT8ew5NjA.", maxpg: ".$this->y5E6XiZB3w2DW4XFY.", pleft*: ".($ELiT8ew5NjA*$m7sJxx1ev[2]+1000). " cmp: ".($p0o_d8t8MHu9ndEDl - $this->num_urls_processed_in_current_batch). " / totc: ".($YY6ZpVykum).", proccur: ".$this->num_urls_processed_in_current_batch. " | numproc: ".$this->num_processed. ", cmp2: ".$this->y5E6XiZB3w2DW4XFY*$m7sJxx1ev[1]. " -> fpar: ".implode(", ", $m7sJxx1ev)."\n",2); $iHlgQjQb2bucfa6qPN = ($rzR9rfqnQiRlGki || $v7cZje3jo);// && !$Xg99Dr7wV27xjesu; $_fp3 = $m7sJxx1ev[3]; if($Xg99Dr7wV27xjesu)$_fp3*=2; if(!$iHlgQjQb2bucfa6qPN) $iHlgQjQb2bucfa6qPN = ($_fp3 && $this->y5E6XiZB3w2DW4XFY && (($YY6ZpVykum>$this->y5E6XiZB3w2DW4XFY*$_fp3))); if($iHlgQjQb2bucfa6qPN){ } if(!$Xg99Dr7wV27xjesu) $U91midzXMtJ = $m7sJxx1ev[3] && $this->y5E6XiZB3w2DW4XFY && (($this->num_processed>$this->y5E6XiZB3w2DW4XFY*$m7sJxx1ev[3])); if($U91midzXMtJ){ $urls_list = $urls_list2 = array(); $this->E7PbBLC9yunfxjh = 0; $this->num_links_current_batch = 0; } $OKRWM1CfS0GVN = ($O5DEYzmlsLOVgGYR1jS<=0 || $this->links_level<$O5DEYzmlsLOVgGYR1jS); if(!$qaauW0uNhxPeh0CAA9z && $OKRWM1CfS0GVN) { RYtkTQfZ1my('extract'); $eip_KPaKUUBFoURGriK = array_sum(explode(' ', microtime())); $wljeCd6fCRKulnoFsC4 = XGuLWSESslGQjVANPx1($this->A78wE6jgmrny, $this->aMX8JQFnFnMWTuOJ); if(wS0f9iXRiSA('xs_http_parallel', false) || $Ar3WLiESdY){ if(!$SMLYdzM8Msiva534J4 && !isset($k0huGF8gg4LQ1A->f2TxlO8iTzsmVAI[$wljeCd6fCRKulnoFsC4])){ $SMLYdzM8Msiva534J4 = array(); $hJyw6kplxZTuBo_ = array(); $_par = wS0f9iXRiSA('xs_http_parallel_num', 10); for($i=0;($i<$_par*5)&&(count($hJyw6kplxZTuBo_)<$_par);$i++) { $_ul = $i ? key($urls_list) : $wljeCd6fCRKulnoFsC4; $_added_toqueue = false; if($_ul) { $_fex2 = $this->k7_EQIP1qhQCfrZAa($_ul); $_allow_u = !$_fex2['f'] && !$_fex2['f2']; if($_allow_u){ if($Ar3WLiESdY){ $_allow_u = !$this->puPBJwgZjs1JB2($_ul); } if($_allow_u){ $_u1 = XGuLWSESslGQjVANPx1($this->A78wE6jgmrny, $_ul); if(!isset($this->sm_base[$_u1])){ $hJyw6kplxZTuBo_[] = $_u1; $_added_toqueue = true; } } } } if($_added_toqueue) { if($i>0){ next($urls_list); // move pointer $SMLYdzM8Msiva534J4[] = array($_ul); } }else // skip scanning if non matching URL found and process as many as found at this point break; } if($hJyw6kplxZTuBo_){ $k0huGF8gg4LQ1A->yOp0Q9sI83Pd5w8($hJyw6kplxZTuBo_); }else { $SMLYdzM8Msiva534J4 = array(); } } } wkwPIGj7HFydE9Mly("<h4> { $wljeCd6fCRKulnoFsC4 } </h4>\n"); $H3psU6E1Oh=0; $m8HTpd_0s++; $UXFc3uMWQ = false; do { $uFxTSS9XSb52KZo7YrD = $k0huGF8gg4LQ1A->fetch($wljeCd6fCRKulnoFsC4, 0, 0); $this->bBAymp71W4mgyv_2c($uFxTSS9XSb52KZo7YrD['last_url'], $this->AKn7hFzTSwEhWIx); $this->KiQEfebckxJVM(); $_to = $uFxTSS9XSb52KZo7YrD['flags']['socket_timeout']; if($_to && ($nqlGKC7b_ZAN7dg['host']!=$uFxTSS9XSb52KZo7YrD['purl']['host'])){ $uFxTSS9XSb52KZo7YrD['flags']['error'] = 'Host doesn\'t match'; } $_ic = intval($uFxTSS9XSb52KZo7YrD['code']); $c0NPvBdlsu = preg_match($this->badreqcode,$_ic); $M3UkKfCl2Ig = $this->interruptcode && preg_match($this->interruptcode,$_ic); $kOOkR04Ndiu = ($_ic == 403); if($this->allowcode && preg_match($this->allowcode,$_ic)){ $c0NPvBdlsu = $kOOkR04Ndiu = false; } $X9DhIsP39Q2Z2Ln_ = (($_ic == 301)||($_ic==302)) && ($wljeCd6fCRKulnoFsC4 == $uFxTSS9XSb52KZo7YrD['last_url']); if($M3UkKfCl2Ig) { $X5O38TAuWv4FR = "Stop processing code [$_ic] - interrupt session"; $UXFc3uMWQ = true; break; } if( !$uFxTSS9XSb52KZo7YrD['flags']['error'] && (($c0NPvBdlsu || $kOOkR04Ndiu || $X9DhIsP39Q2Z2Ln_) || !$uFxTSS9XSb52KZo7YrD['code'] || $_to) ) { $H3psU6E1Oh++; $_sl = $grab_parameters['xs_delay_ms']?$grab_parameters['xs_delay_ms']:1; if($uFxTSS9XSb52KZo7YrD['headers'] && ($_csl = $uFxTSS9XSb52KZo7YrD['headers']['retry-after'])) $_sl = min(120,max($_sl, $_csl + ($H3psU6E1Oh+1)*$_sl)); if(($_to) && $grab_parameters['xs_timeout_break']){ wkwPIGj7HFydE9Mly("<p> # TIMEOUT - $_to #</p>\n"); if($H3psU6E1Oh==3){ if(strstr($_to,'read') ){ wkwPIGj7HFydE9Mly("<p> read200 break?</p>\n"); break; } if($N0PX75BqtEcrjK++>5) { $X5O38TAuWv4FR = "Too many timeouts detected - interrupt session"; $UXFc3uMWQ = true; break; } wkwPIGj7HFydE9Mly("<p> # MULTI TIMEOUT - SHORT TIME BREAK #</p>\n"); $_sl = 60; $H3psU6E1Oh = 0; } } $L1pZpKbDY = $X9DhIsP39Q2Z2Ln_?"(".$wljeCd6fCRKulnoFsC4." => ".$uFxTSS9XSb52KZo7YrD['last_url'].")":"-"; wkwPIGj7HFydE9Mly("<p> # RETRY - ".$uFxTSS9XSb52KZo7YrD['code']." - (code) ".(intval($uFxTSS9XSb52KZo7YrD['code']))." - error(".$uFxTSS9XSb52KZo7YrD['flags']['error'].") self-redir($L1pZpKbDY) badreq($c0NPvBdlsu) forbreq($kOOkR04Ndiu) tmout($_to)# zZz $_sl</p>\n"); sleep($_sl); } else break; }while($H3psU6E1Oh<3); if($UXFc3uMWQ) { $urls_list = array_merge(array($this->aMX8JQFnFnMWTuOJ => $D2aBvrGfxJa6J32gUZJ), $urls_list); break; } $this->fetch_no++; RYtkTQfZ1my('extract', true); RYtkTQfZ1my('analyze'); $this->htTmB_dxmeudFU = array_sum(explode(' ', microtime()))-$eip_KPaKUUBFoURGriK; $this->nettime += $this->htTmB_dxmeudFU; wkwPIGj7HFydE9Mly("<hr>\n[[[ ".$uFxTSS9XSb52KZo7YrD['code']." ]]] - ".number_format($this->htTmB_dxmeudFU,2)."s (".number_format($k0huGF8gg4LQ1A->x97htgfoi5yU,2).' + '.number_format($k0huGF8gg4LQ1A->hOlwU2wM1,2).")\n".var_export($uFxTSS9XSb52KZo7YrD['headers'],1)); $K71jAZIWNQat1l9V = is_array($uFxTSS9XSb52KZo7YrD['headers']) ? strtolower($uFxTSS9XSb52KZo7YrD['headers']['content-type']) : ''; $GMdJHAjT0DeR6EDV67I = $this->PA0Gyvv3UI($K71jAZIWNQat1l9V) || !$K71jAZIWNQat1l9V; if((strstr($K71jAZIWNQat1l9V,'application/') && strstr($K71jAZIWNQat1l9V,'pdf')) ||strstr($K71jAZIWNQat1l9V,'/xml') ||strstr($K71jAZIWNQat1l9V,'/rss') ||strstr($K71jAZIWNQat1l9V,'text/plain') ) { $uFxTSS9XSb52KZo7YrD['content'] = ''; $GMdJHAjT0DeR6EDV67I = true; } $NHjc8vQoRs5p48 = ($grab_parameters['xs_parse_swf'] && strstr($K71jAZIWNQat1l9V, 'shockwave-flash')); $BFocBiqRkw = ($grab_parameters['xs_parse_js'] && strstr($K71jAZIWNQat1l9V, 'javascript')); if($K71jAZIWNQat1l9V && !$GMdJHAjT0DeR6EDV67I && !$NHjc8vQoRs5p48 && !$BFocBiqRkw){ if(!$qaauW0uNhxPeh0CAA9z){ $OgSj2zxK4Qo5Cc = $K71jAZIWNQat1l9V; continue; } } $KcmpZavBWiSGlc = array(); if($this->NxfMP9qoH('xs_crawl_proc_stripslash', $this->oGNbg9vqexWio($this->aMX8JQFnFnMWTuOJ), false)) { $GMdJHAjT0DeR6EDV67I = true; $uFxTSS9XSb52KZo7YrD['content'] = stripslashes($uFxTSS9XSb52KZo7YrD['content']); } if($uFxTSS9XSb52KZo7YrD['code']==404 || ($grab_parameters['xs_force_404'] && preg_match('#'.implode('|',preg_split('#\s+#',$grab_parameters['xs_force_404'])).'#', $this->aMX8JQFnFnMWTuOJ) ) ){ if($this->links_level>0) if(!$grab_parameters['xs_chlog_list_max'] || count($this->urls_404) < $grab_parameters['xs_chlog_list_max']) { $e7q7NgwSQ6EWh5Ap8o = $this->ref_links_tmp2[$this->aMX8JQFnFnMWTuOJ]; if($e7q7NgwSQ6EWh5Ap8o && isset($this->x15LfqXVsH[$e7q7NgwSQ6EWh5Ap8o[0]]) && isset($this->ref_links_list[$e7q7NgwSQ6EWh5Ap8o[0]]) ){ $e7q7NgwSQ6EWh5Ap8o = array_merge($e7q7NgwSQ6EWh5Ap8o, array_slice($this->ref_links_list[$e7q7NgwSQ6EWh5Ap8o[0]],1)); } $this->urls_404[]=array($this->aMX8JQFnFnMWTuOJ, $e7q7NgwSQ6EWh5Ap8o); $this->addedcnt['bl'] = count($this->urls_404); } } $cn = $uFxTSS9XSb52KZo7YrD['content']; $this->tsize+=strlen($cn); if(!$grab_parameters['xs_parse_js']) if($z_uE1_nRkISYKJyg5VK = preg_replace('#<script.*?</script[^>]*>#is', '',$cn)){ $cn = $z_uE1_nRkISYKJyg5VK; } if($z_uE1_nRkISYKJyg5VK = preg_replace('#<style.*?</style>#is', '',$cn)){ $cn = $z_uE1_nRkISYKJyg5VK; } if($z_uE1_nRkISYKJyg5VK = preg_replace('#<!--(\[if IE\]>|.*?-->)#is', '',$cn)){ $cn = $z_uE1_nRkISYKJyg5VK; } wkwPIGj7HFydE9Mly("\n\n<hr>\n\n$cn\n\n<hr>\n\n",4);//exit; preg_match('#<base[^>]*?href=[\'"](.*?)[\'"]#is', $cn, $bm); $this->CkkDVwCnm_1R($bm, $i1zq0dY3F9_zWWq); if($grab_parameters['xs_canonical']) if(($wljeCd6fCRKulnoFsC4 == $uFxTSS9XSb52KZo7YrD['last_url']) && ( preg_match('#<link[^>]*rel=[\'"]canonical[\'"][^>]*\shref=[\'"]([^>]*?)[\'"]#is', $cn, $U6pUvjdja) || preg_match('#<link[^>]*\shref=[\'"]([^>]*?)[\'"][^>]*rel=[\'"]canonical[\'"]#is', $cn, $U6pUvjdja)) ){ $PTdMTgtT4 = $this->drKZs8x1lb3X3n(trim($U6pUvjdja[1])); $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($PTdMTgtT4, $this->aMX8JQFnFnMWTuOJ); if($wljeCd6fCRKulnoFsC4 != $PTdMTgtT4){ $OgSj2zxK4Qo5Cc = 'canonical - '.$PTdMTgtT4; } if(wS0f9iXRiSA('xs_canonical_nofollow', false)) $iHlgQjQb2bucfa6qPN = true; } if($uFxTSS9XSb52KZo7YrD['last_url']){ $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($uFxTSS9XSb52KZo7YrD['last_url'], $this->aMX8JQFnFnMWTuOJ); if($LV3feH44NvQ == 1){ $OgSj2zxK4Qo5Cc = 'lu (ext) - '.$uFxTSS9XSb52KZo7YrD['last_url']; if($uFxTSS9XSb52KZo7YrD['last_url'] != $wljeCd6fCRKulnoFsC4) { continue; } } } $Io29DwFf3WcSyMtAl = $this->KVz31hE2O($uFxTSS9XSb52KZo7YrD['last_url']); if(($wljeCd6fCRKulnoFsC4 != $uFxTSS9XSb52KZo7YrD['last_url']))// && ($wljeCd6fCRKulnoFsC4 != $uFxTSS9XSb52KZo7YrD['last_url'].'/')) { $this->x15LfqXVsH[$this->aMX8JQFnFnMWTuOJ] = $uFxTSS9XSb52KZo7YrD['last_url']; $io=$this->aMX8JQFnFnMWTuOJ; if(strlen($Io29DwFf3WcSyMtAl) <= 2048) if(!isset($urls_list_full[$Io29DwFf3WcSyMtAl])) { $urls_list2[$Io29DwFf3WcSyMtAl]++; $this->VAsjehLyY7SfsjZNoL2($Io29DwFf3WcSyMtAl, $this->aMX8JQFnFnMWTuOJ); } $OgSj2zxK4Qo5Cc = 'lu - '.$uFxTSS9XSb52KZo7YrD['last_url']; if(!$qaauW0uNhxPeh0CAA9z)continue; } if($this->allowcode && !preg_match($this->allowcode,$uFxTSS9XSb52KZo7YrD['code'])){ $OgSj2zxK4Qo5Cc = $uFxTSS9XSb52KZo7YrD['code']; continue; } $retrno++; if($qaauW0uNhxPeh0CAA9z||$iHlgQjQb2bucfa6qPN) { $GMdJHAjT0DeR6EDV67I = false; } RYtkTQfZ1my('analyze',true); if($NHjc8vQoRs5p48) { include_once plQDGddmmXu9xZB.'class.pfile.inc.php'; $am = new SWFParser(); $am->POwAT9n5Jfw($cn); $p0Z32taip_p = $am->o4tgRKj7P07n8(); }else if($GMdJHAjT0DeR6EDV67I || $BFocBiqRkw) { RYtkTQfZ1my('parse'); if($BFocBiqRkw) { $OgSj2zxK4Qo5Cc = 'js'; $f = true; // Do not index preg_match_all('#(?:add|menu)item\s*\([^\)]*?["\']((?:\.+\/|http).*?)["\)\']#is', $cn, $L2HmGWTj9H); $KcmpZavBWiSGlc = $L2HmGWTj9H[1]; if(preg_match_all('#\(\s*(\{.*?["\']content["\']\s*\:\s*[\'"](.*?)[\'"]\s*\})\s*\)#is', $cn, $T9v4StE5HjmB1)){ foreach($T9v4StE5HjmB1[1] as $CYUcx8sQS57c4YY){ if($BJRQ6cQcDwIriiwiv6g = stripslashes($CYUcx8sQS57c4YY)){ $KcmpZavBWiSGlc = array_merge( $KcmpZavBWiSGlc, $this->am05AsFKZIyD($BJRQ6cQcDwIriiwiv6g)); } } } }else { $KcmpZavBWiSGlc = $this->am05AsFKZIyD($cn); } } $KcmpZavBWiSGlc = array_unique($KcmpZavBWiSGlc); $nn = $nt = 0; reset($KcmpZavBWiSGlc); if(isset($grab_parameters['xs_robotstxt']) && $grab_parameters['xs_robotstxt']){ if(preg_match('#<meta[^>]*name=[\'"]robots[\'"][^>]*>#is',$cn,$_cm)) if(preg_match('#nofollow#is',$_cm[0])) $KcmpZavBWiSGlc = array(); } if(!$this->runstate['charset']){ if(preg_match('#<meta[^>]*?charset\s*=\s*"?([^">]*)"#is',$cn, $hPw74OQezczKteg0gC)) $this->runstate['charset'] = $hPw74OQezczKteg0gC[1]; } RYtkTQfZ1my('parse', true); RYtkTQfZ1my('llist'); foreach($KcmpZavBWiSGlc as $i=>$ll) if($ll) { if(preg_match('#^https?%3a%2f#i', $ll)) $ll = urldecode($ll); $a = $sa = trim($ll); $a = str_replace(':',':',$a); if($grab_parameters['xs_proto_skip'] && (preg_match('#^'.$grab_parameters['xs_proto_skip'].'#i',$a)|| ($this->Te7nRdc2NfOAPzQBZFq && preg_match('#'.$grab_parameters['xs_exc_skip'].'#i',$a))|| preg_match('#^'.$grab_parameters['xs_proto_skip'].'#i',function_exists('html_entity_decode')?html_entity_decode($a):$a) )) continue; if(strlen($a) > 4096) continue; $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($a, $this->aMX8JQFnFnMWTuOJ); if($LV3feH44NvQ == 1) { if($grab_parameters['xs_extlinks'] && (!$grab_parameters['xs_extlinks_excl'] || !preg_match('#'.$this->J4vPRMHzK($grab_parameters['xs_extlinks_excl']).'#',$a)) && (!$grab_parameters['xs_ext_max'] || (count($urls_ext)<$grab_parameters['xs_ext_max'])) ) { if(!$urls_ext[$a] && (!$grab_parameters['xs_ext_skip'] || !preg_match('#'.$grab_parameters['xs_ext_skip'].'#',$a) ) ) $urls_ext[$a] = $wljeCd6fCRKulnoFsC4; } continue; } $Io29DwFf3WcSyMtAl = $LV3feH44NvQ ? $a : substr($a,strlen($this->A78wE6jgmrny)); $Io29DwFf3WcSyMtAl = str_replace(' ', '%20', $Io29DwFf3WcSyMtAl); if($grab_parameters['xs_exclude_check']) { $_f=$_f2=false; $_f=$this->MVZvUfEx6bYBJwmz&&preg_match('#('.$this->MVZvUfEx6bYBJwmz.')#',$Io29DwFf3WcSyMtAl); $_f = $_f || ($_fbot = $this->MgKnEUlaw6_XfzaD($this->v0ytSBWER.$Io29DwFf3WcSyMtAl)); if($_f)continue; } if(strcmp($Io29DwFf3WcSyMtAl, $this->aMX8JQFnFnMWTuOJ) == 0) continue; $this->VAsjehLyY7SfsjZNoL2($Io29DwFf3WcSyMtAl, $this->aMX8JQFnFnMWTuOJ); if($urls_list_full[$Io29DwFf3WcSyMtAl]) continue; wkwPIGj7HFydE9Mly("<u>[[[acut] $Io29DwFf3WcSyMtAl ]]]</u><br>\n",2);//exit; $urls_list2[$Io29DwFf3WcSyMtAl]++; $nt++; } unset($KcmpZavBWiSGlc); RYtkTQfZ1my('llist', true); } // if(!$qaauW0uNhxPeh0CAA9z && $OKRWM1CfS0GVN) } // if(!$f && !$f2) $this->E7PbBLC9yunfxjh = count($urls_list2); RYtkTQfZ1my('analyze', true); RYtkTQfZ1my('post'); if(!$f){ if($f = $f || !$this->NxfMP9qoH('xs_incl_only', $this->oGNbg9vqexWio($this->aMX8JQFnFnMWTuOJ), true)) $OgSj2zxK4Qo5Cc = 'inclonly'; } if($_fex['f3']) { $f = true; $OgSj2zxK4Qo5Cc = 'noinclmask'; } if(!$f) if(isset($grab_parameters['xs_robotstxt']) && $grab_parameters['xs_robotstxt']) { if(preg_match('#<meta[^>]*name=[\'"]robots[\'"][^>]*>#is',$cn,$_cm)) $f = preg_match('#noindex#is',$_cm[0]); if($f){ if(strstr($cn, 'Incapsula')) $OgSj2zxK4Qo5Cc = 'incapsula'; else $OgSj2zxK4Qo5Cc = 'mrob'; } } if(!$f && !$OgSj2zxK4Qo5Cc) { if(!$gEBIZ5cHL) { $gEBIZ5cHL = array( 'link' => preg_replace('#//+$#','/', preg_replace('#^([^/\:\?]/)/+#','\\1', (preg_match('#^\w+://#',$this->aMX8JQFnFnMWTuOJ) ? $this->aMX8JQFnFnMWTuOJ : $this->A78wE6jgmrny . $this->aMX8JQFnFnMWTuOJ) )) ); $j0dFHpRFFgv = 1024; if($grab_parameters['xs_makehtml']||$grab_parameters['xs_makeror']||$grab_parameters['xs_rssinfo']||$grab_parameters['xs_newsinfo']) { preg_match('#<title[^>]*?>([^<]*?)</title>#is', $uFxTSS9XSb52KZo7YrD['content'], $h7F8_WaY6lWU5mvC); $gEBIZ5cHL['t'] = substr(trim(strip_tags($h7F8_WaY6lWU5mvC[1])),0,$j0dFHpRFFgv); } if($grab_parameters['xs_metadesc']||$grab_parameters['xs_videoinfo']) { if(!preg_match('#<meta\s[^>]*(?:http-equiv|name)\s*=\s*"?description[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs)) preg_match('#<meta[^>]*content\s*=\s*["]?([^>\"]*)[^>]*?(?:http-equiv|name)\s*=\s*"?description#is', $cn, $MdWOO751erxbvs); if($MdWOO751erxbvs[1]) $gEBIZ5cHL['d'] = substr(trim($MdWOO751erxbvs[1]),0,$j0dFHpRFFgv); } if($grab_parameters['xs_makeror']||$grab_parameters['xs_autopriority']) $gEBIZ5cHL['o'] = max(0,$this->links_level); if($zN7QvVaBZeQp5ZF) $gEBIZ5cHL['p'] = $zN7QvVaBZeQp5ZF; if(preg_match('#<meta\s[^>]*(?:http-equiv|name)\s*=\s*"?last-modified[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs)){ $gEBIZ5cHL['clm'] = str_replace('@',' ',$MdWOO751erxbvs[1]); } if(preg_match('#<meta\s[^>]*(?:http-equiv|name)\s*=\s*"?priority[^>]*content\s*=\s*["]?([\d\.]+)#is', $cn, $MdWOO751erxbvs)){ $gEBIZ5cHL['p'] = $MdWOO751erxbvs[1]; } if(preg_match('#<meta\s[^>]*(?:http-equiv|name)\s*=\s*"?changefreq[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs)){ $gEBIZ5cHL['f'] = $MdWOO751erxbvs[1]; }else if(preg_match('#<meta\s[^>]*(?:http-equiv|name)\s*=\s*"?revisit-after[^>]*content\s*=\s*["]?([^>\"]*)#is', $cn, $MdWOO751erxbvs)){ if(preg_match('#(\d+)\s*hour#',$MdWOO751erxbvs[1])){ $gEBIZ5cHL['f'] = 'hourly'; } if(preg_match('#(\d+)\s*month#',$MdWOO751erxbvs[1])){ $gEBIZ5cHL['f'] = 'monthly'; } if(preg_match('#(\d+)\s*day#',$MdWOO751erxbvs[1], $MdWOO751erxbvs)){ $d = $MdWOO751erxbvs[1]+0; if($d<4)$gEBIZ5cHL['f'] = 'daily'; else if($d<22)$gEBIZ5cHL['f'] = 'weekly'; else $gEBIZ5cHL['f'] = 'monthly'; } } if(preg_match('#'.$Xmfk5aHUVgA7WxET.'#',$this->oGNbg9vqexWio($this->aMX8JQFnFnMWTuOJ),$eADRrzkKihKqzHhKTq7)) { for($_i=0;$_i<count($eADRrzkKihKqzHhKTq7);$_i++) { if($eADRrzkKihKqzHhKTq7[$_i+1]) break; } if($Rcj3xnKl_qu[$_i]) { if(!$gEBIZ5cHL['clm']) $gEBIZ5cHL['clm'] = $Rcj3xnKl_qu[$_i]['lm']; if(!$gEBIZ5cHL['f']) $gEBIZ5cHL['f'] = $Rcj3xnKl_qu[$_i]['f']; $gEBIZ5cHL['p'] = $Rcj3xnKl_qu[$_i]['p']; } } $xz = 'img'; $_maxi = $grab_parameters['xs_img_maxtot']; $_maxi_tot = $_maxi ? $_maxi : $this->y5E6XiZB3w2DW4XFY*100; if(!$_maxi_tot || ($this->addedcnt['i']<$_maxi_tot)) if($grab_parameters['xs_imginfo']) { $_imgext = '(?:jpg|png|gif|jpeg|bmp)'; preg_match_all('#<(?:(?:amp-)?img|div)[^>]*?(?:(?:src|data-(?:image|thumbnail|original))(?:set)?\s*=\s*|image\:\s*url\s*\(\s*)("([^">]+)|\'([^\'>]+)|([^\s\"\\\\>]+))(.*?>)#is', $cn, $dbZYiSonNrlDO42usc, PREG_SET_ORDER); preg_match_all('#<a[^>]*\shref\s*=\s*[\'"]?([^>\'" ]*?\.'.$_imgext.')([\'" ][^>]*?>|>)#is', $cn, $L2HmGWTj9H, PREG_SET_ORDER); $T9v4StE5HjmB1 = array(); $ai = $ae = array(); $dbZYiSonNrlDO42usc = array_merge($dbZYiSonNrlDO42usc, $L2HmGWTj9H, $T9v4StE5HjmB1); $_maxi = $grab_parameters['xs_img_maxperpage']; if(!$_maxi)$_maxi = 1000; if($uFxTSS9XSb52KZo7YrD['inpdata'] && ($_dimg = $uFxTSS9XSb52KZo7YrD['inpdata']['img'])){ foreach($_dimg as $im) { $dbZYiSonNrlDO42usc[] = array('', $im['url']); } } foreach($dbZYiSonNrlDO42usc as $im) { $_il = array($im[1]); if(strstr($im[0],'srcset')){ $_il = preg_split('#(\s*,\s*|\s+[^\./]+\s*)#', $im[1]); } foreach($_il as $_ili) if($_ili) { $_im1 = trim(preg_replace('#^[\'\"]+#','',$_ili)); $LV3feH44NvQ = $this->bBAymp71W4mgyv_2c($_im1, $this->aMX8JQFnFnMWTuOJ, $grab_parameters['xs_img_allow_domains']); if($LV3feH44NvQ == 1) continue; if($grab_parameters['xs_imgincmask']) if(!$this->NxfMP9qoH('xs_imgincmask', $_im1, true)) continue; $XegIxMo9HSXuEr = $this->v0ytSBWER.substr($_im1,strlen($this->A78wE6jgmrny)); if($LV3feH44NvQ != 2) { if($this->MgKnEUlaw6_XfzaD($XegIxMo9HSXuEr)) continue; } if(!$this->MVZvUfEx6bYBJwmz||!@preg_match('#('.$this->MVZvUfEx6bYBJwmz.')#',$XegIxMo9HSXuEr)) { preg_match('#(?:title|alt)\s*=\s*"(.*?)"#i', $im[0], $tm); $t = trim($tm[1]); if(!$imlist[md5($_im1.($grab_parameters['xs_img_hashtitle'] ? '-'.$t : ''))]++) { $ai[] = array($_im1, $t); } } if(count($ai) >= $_maxi)break; } } if($ai){ $gEBIZ5cHL['i'] = $ai; $this->addedcnt['i'] += count($ai); } wkwPIGj7HFydE9Mly("{ images added: ".count($ai)." }<br>\n",2); } $xz = '/img'; $xz = 'video'; $_maxv_tot = $this->y5E6XiZB3w2DW4XFY * 10; if(!$_maxv_tot || ($this->addedcnt['v']<$_maxv_tot)) if($grab_parameters['xs_videoinfo']) { $BJS7s_fNyAAZU26 = array( '(?:https?\:)?//(?:www\.)?youtube(?:-nocookie)?.com/(?:v/|embed/|watch\?v=)', '(?:https?\:)?//youtu\.be/', '(?:https?\:)?//video.google.com/googleplayer\.swf\?docid=', '(?:https?\:)?//(?:player\.)?vimeo.com/(?:video/|moogaloop\.swf\?clip_id=)', '(?:https?\:)?//(?:www\.)?dailymotion.com/(?:embed|swf)/video/', '(?:https?\:)?//(?:media.mtvnservices.com|www.mtv.com/player/)[^>]*?vid(?:=|%3D)', '(?:https?\:)?//blip.tv/play/', '(?:https?\:)?//network.coull.com/(?:embediframe/|embed)\?v=' ); $BJS7s_fNyAAZU26 = '(?:'. implode('|', $BJS7s_fNyAAZU26).')'; $IQXRrpl7X = '[a-z0-9\-\_]'; $BJS7s_fNyAAZU26 = '('.$BJS7s_fNyAAZU26.'('.$IQXRrpl7X.'+))'; preg_match_all('#<param[^>]*name=[\'"\s]*?(?:movie|src)[^>]*value=[\'"\s]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $dbZYiSonNrlDO42usc, PREG_SET_ORDER); preg_match_all('#flashvars\s*=\s*\{[^\}]*?file:\s*[\'"]'.$BJS7s_fNyAAZU26.'#is', $cn, $s2DT0A9dOu, PREG_SET_ORDER); preg_match_all('#<(?:embed|script)[^>]*src=[\'"\s]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $p0Z32taip_p, PREG_SET_ORDER); preg_match_all('#<(?:iframe|video)[^>]*src=[^>]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $m07f5HVKi, PREG_SET_ORDER); preg_match_all('#<[^>]*(?:href|data-path)=[^>]*?'.$BJS7s_fNyAAZU26.'([^>\'"]*)#is', $cn, $yUuYWZ7cq, PREG_SET_ORDER); preg_match_all('#file\"\s*\:\s*\[\'"]'.$BJS7s_fNyAAZU26.'#is', $cn, $vieMCCQX4, PREG_SET_ORDER); preg_match_all('#\.addVariable\([\'\"]file[\'\"],[\'\"]'.$BJS7s_fNyAAZU26.'#is', $cn, $t, PREG_SET_ORDER); preg_match_all('#var\s+youtube\s+=\s+\'('.$IQXRrpl7X.'+)#is', $cn, $_ym1, PREG_SET_ORDER); foreach($_ym1 as $_yv){ $dbZYiSonNrlDO42usc[] = array('', 'http://youtu.be/'.$_yv[1],$_yv[1]); } $ai = $ae = $UTnnUtzvzSy0XAP = array(); preg_match_all('#jwplayer\([^\)]*?\)\.setup\(\{(.*?)\}\)#is', $cn, $ps); if($ps && $ps[1]) { foreach($ps[1] as $lodni7zmzvSYqgs){ if( preg_match('#\"provider\"\s*\:\s*\"youtube\"#is', $lodni7zmzvSYqgs, $m1) && preg_match('#\"file\"\s*\:\s*\"[^\"]*?(?:v=)?('.$IQXRrpl7X.'+)#is', $lodni7zmzvSYqgs, $m2) ){ preg_match('#\"image\"\s*\:\s*\"([^\"]+)#is', $lodni7zmzvSYqgs, $m3); $_th = $m3 ? $m3[1] : ''; if($_th) $this->bBAymp71W4mgyv_2c($_th, $this->aMX8JQFnFnMWTuOJ); $UTnnUtzvzSy0XAP[] = array( '', 'http://www.youtube.com/watch?v='.$m2[1], $m2[1], '', $gEBIZ5cHL['t'], $gEBIZ5cHL['d'], $_th ); } } } preg_match_all('#<div[^>]*class="youtube[^>]*data-id="(.*?)"#is', $cn, $ps); if($ps && $ps[1]) { foreach($ps[1] as $m2){ $UTnnUtzvzSy0XAP[] = array( '', 'http://www.youtube.com/watch?v='.$m2, $m2 ); } } $dbZYiSonNrlDO42usc = array_merge($dbZYiSonNrlDO42usc,$p0Z32taip_p,$m07f5HVKi,$s2DT0A9dOu, $yUuYWZ7cq, $vieMCCQX4, $UTnnUtzvzSy0XAP, $t); preg_match_all('#<[^>]*id="video".*?>(.*?)</div>#is', $cn, $ps); if($ps && $ps[1]) { foreach($ps[1] as $lodni7zmzvSYqgs){ if( preg_match('#<[^>]*href="(.*?)"[^>]*data-type="video"#is', $lodni7zmzvSYqgs, $m_l) && preg_match('#<img[^>]*src="(.*?)"#is', $lodni7zmzvSYqgs, $m_i) ){ $_th = $m_i[1]; $this->bBAymp71W4mgyv_2c($_th, $this->aMX8JQFnFnMWTuOJ); $this->bBAymp71W4mgyv_2c($m_l[1], $this->aMX8JQFnFnMWTuOJ); $ai[] = array( 'title' => strip_tags($gEBIZ5cHL['t']), 'playerloc' => $m_l[1], 'thumb' => $_th, 'desc' => strip_tags($gEBIZ5cHL['d']), ); } } } $ps = preg_split('#<[^>]*itemtype=\"http\:\/\/schema\.org\/VideoObject\"#is', $cn); if(count($ps)>1) { $Agor9azeTRg65OdCMf = array(); foreach($ps as $pp){ if( (preg_match('#<[^>]*itemprop=\"name\"[^>]*content="([^>]*?)"#is', $pp, $m1) ||preg_match('#<[^>]*itemprop=\"name\"[^>]*?>(.*?)<#is', $pp, $m1) ) && preg_match('#<[^>]*itemprop=\"(?:(?:content|embed)(?:url)?)\"[^>]*content="([^>]*?)"#is', $pp, $m2) && preg_match('#<[^>]*itemprop=\"thumbnail(?:url)?\"[^>]*content="([^>]*?)"#is', $pp, $m3) ){ preg_match('#<[^>]*itemprop=\"duration\"[^>]*content="([^>]*?)"#is', $pp, $m11); if(!preg_match('#<[^>]*itemprop=\"description\"[^>]*?content="([^>]*?)"#is', $pp, $m4)) preg_match('#<[^>]*itemprop=\"description\"[^>]*?>(.*?)</(div|h|p)#is', $pp, $m4); $dfUGNFzE8 = $m11[1]; if(preg_match('#T(?:([\d\.]+)H)?(?:([\d\.]+)M)?([\d\.]+)S#i', $dfUGNFzE8, $dm)) { $dfUGNFzE8 = $dm[3] + $dm[2]*60 + intval($dm[1])*3600;; }else if(preg_match('#(?:([\d\.]+)\:)?(?:([\d\.]+)\:)([\d\.]+)S#i', $dfUGNFzE8, $dm)) { $dfUGNFzE8 = $dm[3] + $dm[2]*60 + intval($dm[1])*3600;; } $this->bBAymp71W4mgyv_2c($m2[1], $this->aMX8JQFnFnMWTuOJ, true); $this->bBAymp71W4mgyv_2c($m3[1], $this->aMX8JQFnFnMWTuOJ, true); $_anx = array( 'title' => strip_tags($m1[1]), 'playerloc' => $m2[1], 'thumb' => $m3[1], 'desc' => strip_tags(preg_replace('#<br.*?>#', "\n", ($m4 ? $m4[1] : ($gEBIZ5cHL['d'] ? $gEBIZ5cHL['d'] : $m1[1])))), 'dur' => $dfUGNFzE8, ); if(preg_match('#'.$BJS7s_fNyAAZU26.'#is', $m2[1], $_lm)) $_anx['vid'] = trim($_lm[2]); $Agor9azeTRg65OdCMf[] = $_anx; } } if($Agor9azeTRg65OdCMf){ $ai = $Agor9azeTRg65OdCMf; } } preg_match_all('#<script[^>]*application/ld\+json[^>]*>(.+?)</script>#is', $cn, $ps); if(count($ps)>1) { $Agor9azeTRg65OdCMf = array(); foreach($ps[1] as $pp){ $Z4oarkWIP004duZRulZ = json_decode($pp, true); if($Z4oarkWIP004duZRulZ) foreach($Z4oarkWIP004duZRulZ as $k=>$v)$Z4oarkWIP004duZRulZ[strtolower($k)] = $v; if( ($Z4oarkWIP004duZRulZ['@type'] == 'VideoObject') && $Z4oarkWIP004duZRulZ['embedurl'] && $Z4oarkWIP004duZRulZ['thumbnailurl'] ){ preg_match('#\"duration\"\s*\:\s*\"(.*?)\"#is', $pp, $m11); preg_match('#\"description\"\s*\:\s*\"(.*?)\"#is', $pp, $m4); $dfUGNFzE8 = $Z4oarkWIP004duZRulZ['duration']; if(preg_match('#T(?:([\d\.]+)H)?(?:([\d\.]+)M)?([\d\.]+)S#i', $dfUGNFzE8, $dm)) { $dfUGNFzE8 = $dm[3] + $dm[2]*60 + $dm[1]*3600;; } $this->bBAymp71W4mgyv_2c($m2[1], $this->aMX8JQFnFnMWTuOJ, true); $this->bBAymp71W4mgyv_2c($m3[1], $this->aMX8JQFnFnMWTuOJ, true); $_anx = array( 'title' => strip_tags($Z4oarkWIP004duZRulZ['name']), 'playerloc' => $Z4oarkWIP004duZRulZ['embedurl'], 'thumb' => $Z4oarkWIP004duZRulZ['thumbnailurl'], 'desc' => strip_tags($Z4oarkWIP004duZRulZ['description']), 'dur' => $dfUGNFzE8, ); if(preg_match('#'.$BJS7s_fNyAAZU26.'#is', $Z4oarkWIP004duZRulZ['embedurl'], $_lm)) $_anx['vid'] = trim($_lm[2]); $Agor9azeTRg65OdCMf[] = $_anx; } } if($Agor9azeTRg65OdCMf){ $dbZYiSonNrlDO42usc = array(); $ai = $Agor9azeTRg65OdCMf; } } if(preg_match('#<meta[^>]*property="og:type"[^>]*content="video#is', $cn)) { preg_match('#<meta[^>]*property="og:duration"[^>]*content="(.*?)"#is', $cn, $m1 ); preg_match('#<meta[^>]*property="og:url"[^>]*content="(.*?)"#is', $cn, $m2 ); preg_match('#<meta[^>]*property="og:image"[^>]*content="(.*?)"#is', $cn, $m3 ); preg_match('#<meta[^>]*property="og:title"[^>]*content="(.*?)"#is', $cn, $m4 ); preg_match('#<meta[^>]*property="og:description"[^>]*content="(.*?)"#is', $cn, $m5 ); if($m1&& $m2&& $m3&& $m4) { $cAdokkPjZ4JrKCaE6J0 = explode(':', $m1[1]); $dfUGNFzE8 = $cAdokkPjZ4JrKCaE6J0[0]*60+$cAdokkPjZ4JrKCaE6J0[1]; if(count($cAdokkPjZ4JrKCaE6J0)==3) $dfUGNFzE8 = $dfUGNFzE8*60+$dfUGNFzE8[2]; $dbZYiSonNrlDO42usc = $ai = array(); $_anx = array( 'thumb' => $m3[1], 'title' => $m4[1], 'desc' => $m5?$m5[1]:$m4[1], 'dur' => $dfUGNFzE8, 'playerloc' => $m2[1], ); if(preg_match('#'.$BJS7s_fNyAAZU26.'#is', $m2[1], $_lm)) $_anx['vid'] = trim($_lm[2]); $ai[] = $_anx; } } foreach($dbZYiSonNrlDO42usc as $im) { array_shift($im); $ai[] = $im; } if($ai){ $gEBIZ5cHL['v'] = $ai; } if(function_exists('xml_extra_video_step1')) xml_extra_video_step1($gEBIZ5cHL, $cn, $imlist); } if($_ev = $gEBIZ5cHL['v']){ $ai = array(); foreach($_ev as $im) { if($_vid = $im['vid'] ?$im['vid'] : ($im['playerloc']?$im['playerloc']:$im[1])){ $_i5 = md5(trim($_vid)); $imlist[$_i5]++; if(!$imlist2[$_i5]++) $ai[] = $im; } } $gEBIZ5cHL['v'] = $ai; $this->addedcnt['v'] += count($ai); wkwPIGj7HFydE9Mly("{ videos added: ".count($ai)." }<br>\n",2); } $xz = '/video'; if($grab_parameters['xs_hreflang']){ if( preg_match_all('#<link[^>]*rel\s*=\s*"alternate"[^>]*>#is', $cn, $G9M1u0W69Tn7MIsQ4, PREG_SET_ORDER) ){ $_la = array(); foreach($G9M1u0W69Tn7MIsQ4 as $_alt1){ if(preg_match('#\s(hreflang|media)\s*=\s*"([^">]*?)"[^>]*>#is', $_alt1[0], $_alt) && preg_match('#\s*href\s*=\s*[\'"]([^>]*?)[\'"]#is', $_alt1[0], $_hm) ) { $this->bBAymp71W4mgyv_2c($_hm[1], $this->aMX8JQFnFnMWTuOJ); $_la[] = array('t' => $_alt[1], 'l' => $_alt[2], 'u' => $_hm[1]); } } $gEBIZ5cHL['hl'] = $_la; } } if($grab_parameters['xs_lastmod_notparsed'] && $f2) { $uFxTSS9XSb52KZo7YrD = $k0huGF8gg4LQ1A->fetch($wljeCd6fCRKulnoFsC4, 0, 1, false, "", array('req'=>'HEAD')); } if(!$gEBIZ5cHL['lm'] && isset($uFxTSS9XSb52KZo7YrD['headers']['last-modified'])) $gEBIZ5cHL['lm'] = $uFxTSS9XSb52KZo7YrD['headers']['last-modified']; } if(!$gEBIZ5cHL['lm'] && preg_match('#<meta[^>]*?name\s*=\s*\"last-modified\" content\s*=\s*"?([^">]*)"#is',$uFxTSS9XSb52KZo7YrD['content'], $hPw74OQezczKteg0gC)) $gEBIZ5cHL['lm'] = $hPw74OQezczKteg0gC[1]; RYtkTQfZ1my('post', true); RYtkTQfZ1my('post-save1'); wkwPIGj7HFydE9Mly("\n((include ".$gEBIZ5cHL['link']."))<br />\n"); $wyhMGdq7QGd6 = true; if($grab_parameters['xs_memsave']) { DSdHxXD2UR($oO9pt9QoVu0fjOUEU6, $gEBIZ5cHL); $this->urls_completed[] = $oO9pt9QoVu0fjOUEU6; } else $this->urls_completed[] = $this->dtRHNsRCOCS7gtKcKk3($gEBIZ5cHL); $this->lOXW9WDJYx6ROZX++; RYtkTQfZ1my('post-save1',true); RYtkTQfZ1my('post-save2'); if($grab_parameters['xs_prev_sm_base'] && $this->basecachemask && preg_match('#('.$this->basecachemask.')#',$this->aMX8JQFnFnMWTuOJ)){ $this->sm_base[$this->aMX8JQFnFnMWTuOJ] = $gEBIZ5cHL; } $ELiT8ew5NjA = $this->y5E6XiZB3w2DW4XFY - $this->lOXW9WDJYx6ROZX; RYtkTQfZ1my('post-save2',true); } }while(false);// zero-while RYtkTQfZ1my('post-progress1'); if($this->num_urls_processed_in_current_batch>=$this->num_links_current_batch) { $this->links_level++; wkwPIGj7HFydE9Mly("\n<br>NEXT LEVEL:$this->links_level ($this->num_urls_processed_in_current_batch>=$this->num_links_current_batch)<br />\n"); unset($urls_list); $this->num_urls_processed_in_current_batch = 0; $urls_list = $urls_list2; reset($urls_list); $urls_list_full += $urls_list; $this->num_links_current_batch = count($urls_list); unset($this->ref_links_tmp2); $this->ref_links_tmp2 = $this->ref_links_tmp; unset($this->ref_links_tmp); unset($urls_list2); $this->ref_links_tmp = array(); $urls_list2 = array(); } if(!$wyhMGdq7QGd6){ wkwPIGj7HFydE9Mly("\n({skipped ".$this->aMX8JQFnFnMWTuOJ." - $OgSj2zxK4Qo5Cc})<br />\n"); if(!$grab_parameters['xs_chlog_list_max'] || count($urls_list_skipped) < $grab_parameters['xs_chlog_list_max']) { $urls_list_skipped[$this->aMX8JQFnFnMWTuOJ] = $OgSj2zxK4Qo5Cc; } } RYtkTQfZ1my('post-progress1',true); RYtkTQfZ1my('post-progress2'); $this->num_processed++; R1r_jX8NavH(); $this->pl=min($this->num_links_current_batch - $this->num_urls_processed_in_current_batch,$ELiT8ew5NjA); $eG1Pm653b = ($this->lOXW9WDJYx6ROZX>=$this->y5E6XiZB3w2DW4XFY) || ($this->num_urls_processed_in_current_batch >= $this->num_links_current_batch); if(!$X5O38TAuWv4FR) { if($X5O38TAuWv4FR = VzipJxoohDvbtWM()){ if(!@cMo3XWEqPThU4Id(pbAiQcHEGrGKkyqo4Q.guXHCq5aVeE)) $X5O38TAuWv4FR=0; } } RYtkTQfZ1my('post-progress2',true); RYtkTQfZ1my('post-progress3'); $progpar = $this->KiQEfebckxJVM(false, $eG1Pm653b||$X5O38TAuWv4FR); RYtkTQfZ1my('post-progress3',true); RYtkTQfZ1my('post-progress4'); if($grab_parameters['xs_exec_time'] && ((time()-$dgHgts6lFbG) > $grab_parameters['xs_exec_time']) ){ $X5O38TAuWv4FR = 'Time limit exceeded - '.($grab_parameters['xs_exec_time']).' - '.(time()-$dgHgts6lFbG); } if($grab_parameters['xs_savestate_time']>0 && ( ($this->ctime-$epcQYfCl1>$grab_parameters['xs_savestate_time']) || $eG1Pm653b || $X5O38TAuWv4FR ) ) { $epcQYfCl1 = $this->ctime; wkwPIGj7HFydE9Mly("(saving dump)<br />\n"); $mFUcZxJGA4itwRbJ = array(); $gHdDA3umqlMQEf47qO = array( 'num_urls_processed_in_current_batch', 'urls_list','urls_list2','num_links_current_batch', 'ref_links_tmp','ref_links_tmp2','ref_links_list', 'urls_list_full','urls_completed', 'urls_404', 'nt','tsize','num_processed','links_level','ctime', 'urls_ext','fetch_no', 'starttime', 'retrno', 'nettime', 'urls_list_skipped', 'imlist', 'imlist2', 'progpar', 'runstate', 'sm_sessions', 'addedcnt' ); foreach($gHdDA3umqlMQEf47qO as $k){ $mFUcZxJGA4itwRbJ[$k] = isset($this->$k) ? $this->$k : $$k; } $mFUcZxJGA4itwRbJ['time']=time(); $y5GpEuKtv9=Ie0qgN_fQ8HA($mFUcZxJGA4itwRbJ); Ndm7I4IRr(TYR4q027D0OWvh,$y5GpEuKtv9,pbAiQcHEGrGKkyqo4Q,true); unset($mFUcZxJGA4itwRbJ); unset($y5GpEuKtv9); } if($grab_parameters['xs_delay_req'] && $grab_parameters['xs_delay_ms'] && (($m8HTpd_0s-$WCC6wCSTlQ4MjKyNm3)==$grab_parameters['xs_delay_req'])) { $WCC6wCSTlQ4MjKyNm3 = $m8HTpd_0s; sleep(intval($grab_parameters['xs_delay_ms'])); } RYtkTQfZ1my('post-progress4', true); }while(!$eG1Pm653b && !$X5O38TAuWv4FR); wkwPIGj7HFydE9Mly("\n\n<br><br>Crawling completed<br>\n"); if($_GET['ddbgexit']){ echo '<hr><hr><h2>Dbg exit</h2>'; echo $k0huGF8gg4LQ1A->K7DRsNrsbdnzu9hi.' / '.$k0huGF8gg4LQ1A->nettime.'<hr>'; echo DfjNmPlYlDtVYeKoRsL().'<hr>'; exit; } return array( 'u404'=>$this->urls_404, 'ref_links_list'=>$this->ref_links_list, 'starttime'=>$starttime, 'topmu' => $K5qP9_ZsHx, 'ctime'=>$this->ctime, 'tsize'=>$this->tsize, 'retrno' => $retrno, 'nettime' => $this->nettime, 'errmsg'=>'', 'initurl'=>$this->wOd8ZRUj4KujS, 'initdir'=>$this->A78wE6jgmrny, 'ucount'=>$this->lOXW9WDJYx6ROZX, 'crcount'=>$this->num_processed, 'fetch_no'=>$this->fetch_no, 'time'=>time(), 'params'=>$this->SVH4BzIyh56a_HEGqt, 'sm_sessions'=>$this->sm_sessions, 'interrupt'=>$X5O38TAuWv4FR, 'runstate' => $this->runstate, 'urls_ext'=>$urls_ext, 'urls_list_skipped' => $urls_list_skipped, 'max_reached' => $this->lOXW9WDJYx6ROZX>=$this->y5E6XiZB3w2DW4XFY ); } } $AIqYzQK_mhY = new SiteCrawler(); function xvQcTIF8B113J(){ @cMo3XWEqPThU4Id(pbAiQcHEGrGKkyqo4Q.crI8G26xJzM); if(@file_exists(pbAiQcHEGrGKkyqo4Q.m093Bbc4Eg)) @rename(pbAiQcHEGrGKkyqo4Q.m093Bbc4Eg,pbAiQcHEGrGKkyqo4Q.crI8G26xJzM); }