geturl.lib.php 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361
  1. <?php
  2. /* Copyright (C) 2008-2020 Laurent Destailleur <eldy@users.sourceforge.net>
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation; either version 3 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program. If not, see <https://www.gnu.org/licenses/>.
  16. * or see https://www.gnu.org/
  17. */
  18. /**
  19. * \file htdocs/core/lib/geturl.lib.php
  20. * \brief This file contains functions dedicated to get URLs.
  21. */
  22. /**
  23. * Function to get a content from an URL (use proxy if proxy defined).
  24. * Support Dolibarr setup for timeout and proxy.
  25. * Enhancement of CURL to add an anti SSRF protection:
  26. * - you can set MAIN_SECURITY_ANTI_SSRF_SERVER_IP to set static ip of server
  27. * - common local lookup ips like 127.*.*.* are automatically added
  28. *
  29. * @param string $url URL to call.
  30. * @param string $postorget 'POST', 'GET', 'HEAD', 'PUT', 'PUTALREADYFORMATED', 'POSTALREADYFORMATED', 'DELETE'
  31. * @param string $param Parameters of URL (x=value1&y=value2) or may be a formated content with $postorget='PUTALREADYFORMATED'
  32. * @param integer $followlocation 0=Do not follow, 1=Follow location.
  33. * @param string[] $addheaders Array of string to add into header. Example: ('Accept: application/xrds+xml', ....)
  34. * @param string[] $allowedschemes List of schemes that are allowed ('http' + 'https' only by default)
  35. * @param int $localurl 0=Only external URL are possible, 1=Only local URL, 2=Both external and local URL are allowed.
  36. * @param int $ssl_verifypeer -1=Auto (no ssl check on dev, check on prod), 0=No ssl check, 1=Always ssl check
  37. * @return array Returns an associative array containing the response from the server array('http_code'=>http response code, 'content'=>response, 'curl_error_no'=>errno, 'curl_error_msg'=>errmsg...)
  38. */
  39. function getURLContent($url, $postorget = 'GET', $param = '', $followlocation = 1, $addheaders = array(), $allowedschemes = array('http', 'https'), $localurl = 0, $ssl_verifypeer = -1)
  40. {
  41. //declaring of global variables
  42. global $conf;
  43. $USE_PROXY = empty($conf->global->MAIN_PROXY_USE) ? 0 : $conf->global->MAIN_PROXY_USE;
  44. $PROXY_HOST = empty($conf->global->MAIN_PROXY_HOST) ? 0 : $conf->global->MAIN_PROXY_HOST;
  45. $PROXY_PORT = empty($conf->global->MAIN_PROXY_PORT) ? 0 : $conf->global->MAIN_PROXY_PORT;
  46. $PROXY_USER = empty($conf->global->MAIN_PROXY_USER) ? 0 : $conf->global->MAIN_PROXY_USER;
  47. $PROXY_PASS = empty($conf->global->MAIN_PROXY_PASS) ? 0 : $conf->global->MAIN_PROXY_PASS;
  48. dol_syslog("getURLContent postorget=".$postorget." URL=".$url." param=".$param);
  49. //setting the curl parameters.
  50. $ch = curl_init();
  51. /*print $API_Endpoint."-".$API_version."-".$PAYPAL_API_USER."-".$PAYPAL_API_PASSWORD."-".$PAYPAL_API_SIGNATURE."<br>";
  52. print $USE_PROXY."-".$gv_ApiErrorURL."<br>";
  53. print $nvpStr;
  54. exit;*/
  55. curl_setopt($ch, CURLOPT_VERBOSE, 1);
  56. curl_setopt($ch, CURLOPT_USERAGENT, 'Dolibarr geturl function');
  57. // We use @ here because this may return warning if safe mode is on or open_basedir is on (following location is forbidden when safe mode is on).
  58. // We force value to false so we will manage redirection ourself later.
  59. @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
  60. if (is_array($addheaders) && count($addheaders)) {
  61. curl_setopt($ch, CURLOPT_HTTPHEADER, $addheaders);
  62. }
  63. curl_setopt($ch, CURLINFO_HEADER_OUT, true); // To be able to retrieve request header and log it
  64. // By default use tls decied by PHP.
  65. // You can force, if supported a version like TLSv1 or TLSv1.2
  66. if (!empty($conf->global->MAIN_CURL_SSLVERSION)) {
  67. curl_setopt($ch, CURLOPT_SSLVERSION, $conf->global->MAIN_CURL_SSLVERSION);
  68. }
  69. //curl_setopt($ch, CURLOPT_SSLVERSION, 6); for tls 1.2
  70. // Turning on or off the ssl target certificate
  71. if ($ssl_verifypeer < 0) {
  72. global $dolibarr_main_prod;
  73. $ssl_verifypeer = ($dolibarr_main_prod ? true : false);
  74. }
  75. if (!empty($conf->global->MAIN_CURL_DISABLE_VERIFYPEER)) {
  76. $ssl_verifypeer = 0;
  77. }
  78. // Turning off the server and peer verification(TrustManager Concept).
  79. curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, ($ssl_verifypeer ? true : false));
  80. curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
  81. // Restrict use to some protocols only
  82. $protocols = 0;
  83. if (is_array($allowedschemes)) {
  84. foreach ($allowedschemes as $allowedscheme) {
  85. if ($allowedscheme == 'http') {
  86. $protocols |= CURLPROTO_HTTP;
  87. }
  88. if ($allowedscheme == 'https') {
  89. $protocols |= CURLPROTO_HTTPS;
  90. }
  91. }
  92. curl_setopt($ch, CURLOPT_PROTOCOLS, $protocols);
  93. curl_setopt($ch, CURLOPT_REDIR_PROTOCOLS, $protocols);
  94. }
  95. curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, empty($conf->global->MAIN_USE_CONNECT_TIMEOUT) ? 5 : $conf->global->MAIN_USE_CONNECT_TIMEOUT);
  96. curl_setopt($ch, CURLOPT_TIMEOUT, empty($conf->global->MAIN_USE_RESPONSE_TIMEOUT) ? 30 : $conf->global->MAIN_USE_RESPONSE_TIMEOUT);
  97. //curl_setopt($ch, CURLOPT_SAFE_UPLOAD, true); // PHP 5.5
  98. curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); // We want response
  99. if ($postorget == 'POST') {
  100. curl_setopt($ch, CURLOPT_POST, 1); // POST
  101. curl_setopt($ch, CURLOPT_POSTFIELDS, $param); // Setting param x=a&y=z as POST fields
  102. } elseif ($postorget == 'POSTALREADYFORMATED') {
  103. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); // HTTP request is 'POST' but param string is taken as it is
  104. curl_setopt($ch, CURLOPT_POSTFIELDS, $param); // param = content of post, like a xml string
  105. } elseif ($postorget == 'PUT') {
  106. $array_param = null;
  107. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT'); // HTTP request is 'PUT'
  108. if (!is_array($param)) {
  109. parse_str($param, $array_param);
  110. } else {
  111. dol_syslog("parameter param must be a string", LOG_WARNING);
  112. $array_param = $param;
  113. }
  114. curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($array_param)); // Setting param x=a&y=z as PUT fields
  115. } elseif ($postorget == 'PUTALREADYFORMATED') {
  116. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT'); // HTTP request is 'PUT'
  117. curl_setopt($ch, CURLOPT_POSTFIELDS, $param); // param = content of post, like a xml string
  118. } elseif ($postorget == 'HEAD') {
  119. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'HEAD'); // HTTP request is 'HEAD'
  120. curl_setopt($ch, CURLOPT_NOBODY, true);
  121. } elseif ($postorget == 'DELETE') {
  122. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); // POST
  123. } else {
  124. curl_setopt($ch, CURLOPT_POST, 0); // GET
  125. }
  126. //if USE_PROXY constant set at begin of this method.
  127. if ($USE_PROXY) {
  128. dol_syslog("getURLContent set proxy to ".$PROXY_HOST.":".$PROXY_PORT." - ".$PROXY_USER.":".$PROXY_PASS);
  129. //curl_setopt ($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); // Curl 7.10
  130. curl_setopt($ch, CURLOPT_PROXY, $PROXY_HOST.":".$PROXY_PORT);
  131. if ($PROXY_USER) {
  132. curl_setopt($ch, CURLOPT_PROXYUSERPWD, $PROXY_USER.":".$PROXY_PASS);
  133. }
  134. }
  135. $newUrl = $url;
  136. $maxRedirection = 5;
  137. $info = array();
  138. $response = '';
  139. do {
  140. if ($maxRedirection < 1) {
  141. break;
  142. }
  143. curl_setopt($ch, CURLOPT_URL, $newUrl);
  144. // Parse $newUrl
  145. $newUrlArray = parse_url($newUrl);
  146. $hosttocheck = $newUrlArray['host'];
  147. $hosttocheck = str_replace(array('[', ']'), '', $hosttocheck); // Remove brackets of IPv6
  148. // Deny some reserved host names
  149. if (in_array($hosttocheck, array('metadata.google.internal'))) {
  150. $info['http_code'] = 400;
  151. $info['content'] = 'Error bad hostname '.$hosttocheck.' (Used by Google metadata). This value for hostname is not allowed.';
  152. break;
  153. }
  154. // Clean host name $hosttocheck to convert it into an IP $iptocheck
  155. if (in_array($hosttocheck, array('localhost', 'localhost.domain'))) {
  156. $iptocheck = '127.0.0.1';
  157. } elseif (in_array($hosttocheck, array('ip6-localhost', 'ip6-loopback'))) {
  158. $iptocheck = '::1';
  159. } else {
  160. // Resolve $hosttocheck to get the IP $iptocheck and set CURLOPT_CONNECT_TO to use this ip so curl will not try another resolution that may give a different result
  161. if (function_exists('gethostbyname')) {
  162. $iptocheck = gethostbyname($hosttocheck);
  163. } else {
  164. $iptocheck = $hosttocheck;
  165. }
  166. // TODO Resolve ip v6
  167. }
  168. // Check $iptocheck is an IP (v4 or v6), if not clear value.
  169. if (!filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 | FILTER_FLAG_IPV6)) { // This is not an IP, we clean data
  170. $iptocheck = '0'; //
  171. }
  172. if ($iptocheck) {
  173. if ($localurl == 0) { // Only external url allowed (dangerous, may allow to get malware)
  174. if (!filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
  175. // Deny ips like 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 0.0.0.0/8, 169.254.0.0/16, 127.0.0.0/8 et 240.0.0.0/4, ::1/128, ::/128, ::ffff:0:0/96, fe80::/10...
  176. $info['http_code'] = 400;
  177. $info['content'] = 'Error bad hostname IP (private or reserved range). Must be an external URL.';
  178. break;
  179. }
  180. if (!empty($_SERVER["SERVER_ADDR"]) && $iptocheck == $_SERVER["SERVER_ADDR"]) {
  181. $info['http_code'] = 400;
  182. $info['content'] = 'Error bad hostname IP (IP is a local IP). Must be an external URL.';
  183. break;
  184. }
  185. if (!empty($conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP) && in_array($iptocheck, explode(',', $conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP))) {
  186. $info['http_code'] = 400;
  187. $info['content'] = 'Error bad hostname IP (IP is a local IP defined into MAIN_SECURITY_SERVER_IP). Must be an external URL.';
  188. break;
  189. }
  190. }
  191. if ($localurl == 1) { // Only local url allowed (dangerous, may allow to get metadata on server or make internal port scanning)
  192. // Deny ips NOT like 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 0.0.0.0/8, 169.254.0.0/16, 127.0.0.0/8 et 240.0.0.0/4, ::1/128, ::/128, ::ffff:0:0/96, fe80::/10...
  193. if (filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
  194. $info['http_code'] = 400;
  195. $info['content'] = 'Error bad hostname '.$iptocheck.'. Must be a local URL.';
  196. break;
  197. }
  198. if (!empty($conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP) && !in_array($iptocheck, explode(',', $conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP))) {
  199. $info['http_code'] = 400;
  200. $info['content'] = 'Error bad hostname IP (IP is not a local IP defined into list MAIN_SECURITY_SERVER_IP). Must be a local URL in allowed list.';
  201. break;
  202. }
  203. }
  204. // Common check on ip (local and external)
  205. $arrayofmetadataserver = array('100.100.100.200' => 'Alibaba', '192.0.0.192'=> 'Oracle', '192.80.8.124'=>'Packet');
  206. foreach ($arrayofmetadataserver as $ipofmetadataserver => $nameofmetadataserver) {
  207. if ($iptocheck == $ipofmetadataserver) {
  208. $info['http_code'] = 400;
  209. $info['content'] = 'Error bad hostname IP (Used by '.$nameofmetadataserver.' metadata server). This IP is forbidden.';
  210. break 2; // exit the foreach and the do...
  211. }
  212. }
  213. // Set CURLOPT_CONNECT_TO so curl will not try another resolution that may give a different result. Possible only on PHP v7+
  214. if (defined('CURLOPT_CONNECT_TO')) {
  215. $connect_to = array(sprintf("%s:%d:%s:%d", $newUrlArray['host'], empty($newUrlArray['port'])?'':$newUrlArray['port'], $iptocheck, empty($newUrlArray['port'])?'':$newUrlArray['port']));
  216. //var_dump($newUrlArray);
  217. //var_dump($connect_to);
  218. curl_setopt($ch, CURLOPT_CONNECT_TO, $connect_to);
  219. }
  220. }
  221. // Getting response from server
  222. $response = curl_exec($ch);
  223. $info = curl_getinfo($ch); // Reading of request must be done after sending request
  224. $http_code = $info['http_code'];
  225. if ($followlocation && ($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307)) {
  226. $newUrl = $info['redirect_url'];
  227. $maxRedirection--;
  228. // TODO Use $info['local_ip'] and $info['primary_ip'] ?
  229. continue;
  230. } else {
  231. $http_code = 0;
  232. }
  233. } while ($http_code);
  234. $request = curl_getinfo($ch, CURLINFO_HEADER_OUT); // Reading of request must be done after sending request
  235. dol_syslog("getURLContent request=".$request);
  236. if (!empty($conf->global->MAIN_GETURLCONTENT_OUTPUT_RESPONSE)) {
  237. // This may contains binary data, so we dont output reponse by default.
  238. dol_syslog("getURLContent response =".$response);
  239. }
  240. dol_syslog("getURLContent response size=".strlen($response)); // This may contains binary data, so we dont output it
  241. $rep = array();
  242. if (curl_errno($ch)) {
  243. // Add keys to $rep
  244. $rep['content'] = $response;
  245. // moving to display page to display curl errors
  246. $rep['curl_error_no'] = curl_errno($ch);
  247. $rep['curl_error_msg'] = curl_error($ch);
  248. dol_syslog("getURLContent response array is ".join(',', $rep));
  249. } else {
  250. //$info = curl_getinfo($ch);
  251. // Add keys to $rep
  252. $rep = $info;
  253. //$rep['header_size']=$info['header_size'];
  254. //$rep['http_code']=$info['http_code'];
  255. dol_syslog("getURLContent http_code=".$rep['http_code']);
  256. // Add more keys to $rep
  257. if ($response) {
  258. $rep['content'] = $response;
  259. }
  260. $rep['curl_error_no'] = '';
  261. $rep['curl_error_msg'] = '';
  262. }
  263. //closing the curl
  264. curl_close($ch);
  265. return $rep;
  266. }
  267. /**
  268. * Function get second level domain name.
  269. * For example: https://www.abc.mydomain.com/dir/page.html return 'mydomain'
  270. *
  271. * @param string $url Full URL.
  272. * @param int $mode 0=return 'mydomain', 1=return 'mydomain.com', 2=return 'abc.mydomain.com'
  273. * @return string Returns domaine name
  274. */
  275. function getDomainFromURL($url, $mode = 0)
  276. {
  277. $tmpdomain = preg_replace('/^https?:\/\//i', '', $url); // Remove http(s)://
  278. $tmpdomain = preg_replace('/\/.*$/i', '', $tmpdomain); // Remove part after domain
  279. if ($mode == 2) {
  280. $tmpdomain = preg_replace('/^.*\.([^\.]+)\.([^\.]+)\.([^\.]+)$/', '\1.\2.\3', $tmpdomain); // Remove part 'www.' before 'abc.mydomain.com'
  281. } else {
  282. $tmpdomain = preg_replace('/^.*\.([^\.]+)\.([^\.]+)$/', '\1.\2', $tmpdomain); // Remove part 'www.abc.' before 'mydomain.com'
  283. }
  284. if (empty($mode)) {
  285. $tmpdomain = preg_replace('/\.[^\.]+$/', '', $tmpdomain); // Remove first level domain (.com, .net, ...)
  286. }
  287. return $tmpdomain;
  288. }
  289. /**
  290. * Function root url from a long url
  291. * For example: https://www.abc.mydomain.com/dir/page.html return 'https://www.abc.mydomain.com'
  292. * For example: http://www.abc.mydomain.com/ return 'https://www.abc.mydomain.com'
  293. *
  294. * @param string $url Full URL.
  295. * @return string Returns root url
  296. */
  297. function getRootURLFromURL($url)
  298. {
  299. $prefix = '';
  300. $tmpurl = $url;
  301. $reg = null;
  302. if (preg_match('/^(https?:\/\/)/i', $tmpurl, $reg)) {
  303. $prefix = $reg[1];
  304. }
  305. $tmpurl = preg_replace('/^https?:\/\//i', '', $tmpurl); // Remove http(s)://
  306. $tmpurl = preg_replace('/\/.*$/i', '', $tmpurl); // Remove part after domain
  307. return $prefix.$tmpurl;
  308. }
  309. /**
  310. * Function to remove comments into HTML content
  311. *
  312. * @param string $content Text content
  313. * @return string Returns text without HTML comments
  314. */
  315. function removeHtmlComment($content)
  316. {
  317. $content = preg_replace('/<!--[^\-]+-->/', '', $content);
  318. return $content;
  319. }